fix bugs

.github/workflows/bayes.yml

@@ -4,22 +4,19 @@
 name: bayes:0.0
 
 on:
-  push:
-    branches:
-      - 'main'
   workflow_dispatch:
   schedule:
     # * is a special character in YAML so you have to quote this string : 
 #    weekly #m h d/m m d/w 
 #    - cron:  '30 1 * * 1'
 #    every month 3.rd day
-    - cron:  '40 8 3 * *' 
+    - cron:  '41 8 3 * *' 
 
 jobs:
   docker-run-action:
     runs-on: ubuntu-latest
     container:
-      image: tamboraorg/crecoding:2020.0
+      image: credocker/crecoding:2024.0
       volumes:
         - ${{ github.workspace }}:/cre/python
     steps:
@@ -33,9 +30,9 @@ jobs:
       - name: cd /cre/python/
         run: (cd /cre/python/)
       - name: Install HanTa textblob_de
-        run: (pip3 install HanTa textblob_de)
+        run: (uv pip install HanTa textblob==0.15.3 textblob_de==0.4.3 scikit-learn) 
       - name: Run bayes
-        run: (python3 bayes.py)
+        run: (uv run python bayes.py)
       - name: Submit changes       
         uses: EndBug/add-and-commit@v9
         with:

.github/workflows/manually.yml → .github/workflows/check.yml

@@ -1,15 +1,15 @@
 # This is a basic workflow to help you get started with Actions
 
-name: manually:0.0
+name: check:0.0
 
 on:
   workflow_dispatch:
-
+  
 jobs:
   docker-run-action:
     runs-on: ubuntu-latest
     container:
-      image: tamboraorg/crecoding:2020.0
+      image: credocker/crecoding:2024.0
       env:
         NEWSAPI_KEY: ${{ secrets.NEWSAPI_KEY }}
       volumes:
@@ -20,23 +20,25 @@ jobs:
         run: |
           # this is to fix GIT not liking owner of the checkout dir
           chown -R $(id -u):$(id -g) $PWD
-#        with:
-#          ref: master
-      - name: install aiohttp
-        run: (pip3 install aiohttp)
-      - name: install asyncio
-        run: (pip3 install asyncio)
+#      - name: install aiohttp
+#        run: (pip3 install aiohttp)
+#      - name: install asyncio
+#        run: (pip3 install asyncio)
       - name: cd /cre/python/
         run: (cd /cre/python/)
       - name: cp mysecrets.py
         run: (cp mysecrets.orig.py mysecrets.py)
-      - name: Run newsapi
+      - name: Run check_api
         env:
-            NEWSAPI_KEY: ${{ secrets.NEWSAPI_KEY }}
-        run: (python3 manually.py)
+            GITHUB_OWNER: ${{ github.repository_owner }}
+            GITHUB_REPO:  ${{ github.repository }}
+            NEWSAPI_KEY:  ${{ secrets.NEWSAPI_KEY }}
+            RAPIDAPI_KEY: ${{ secrets.RAPIDAPI_KEY }}
+            GEONAMES_KEY: ${{ secrets.GEONAMES_KEY }}            
+        run: (uv run python check_api.py)
       - name: Submit changes       
         uses: EndBug/add-and-commit@v9
         with:
 #          pull: '--rebase --autostash ...'
-          add: 'csv/*.csv'
+          add: 'CHECK.md'
           tag_push: '--force'

.github/workflows/diagrams.yml

@@ -13,13 +13,13 @@ on:
 #    weekly #m h d/m m d/w 
 #    - cron:  '30 1 * * 1'
 #    daily #m h d/m m d/w 
-    - cron:  '30 1 * * *'
+    - cron:  '50 1 * * *'
 
 jobs:
   docker-run-action:
     runs-on: ubuntu-latest
     container:
-      image: tamboraorg/crecoding:2020.0
+      image: credocker/crecoding:2024.0
       volumes:
         - ${{ github.workspace }}:/cre/python
     steps:
@@ -32,8 +32,10 @@ jobs:
 #          ref: main
       - name: cd /cre/python/
         run: (cd /cre/python/)
+      - name: Install nltk
+        run: (uv pip install nltk scikit-learn)
       - name: Run diagrams
-        run: (python3 diagrams.py)
+        run: (uv run python diagrams.py)
       - name: Submit changes       
         uses: EndBug/add-and-commit@v9
         with:

.github/workflows/entities.yml

@@ -4,22 +4,21 @@
 name: entities:0.0
 
 on:
-  push:
-    branches:
-      - 'main'
   workflow_dispatch:
   schedule:
     # * is a special character in YAML so you have to quote this string : 
 #    weekly #m h d/m m d/w 
-    - cron:  '10 1 * * 1'
+    - cron:  '20 5 * * 1'
 #    daily #m h d/m m d/w 
 #    - cron:  '30 1 * * *'
 
 jobs:
   docker-run-action:
     runs-on: ubuntu-latest
     container:
-      image: tamboraorg/crecoding:2020.0
+      image: credocker/crepython:2020.0
+      env:
+        GEONAMES_KEY: ${{ secrets.GEONAMES_KEY }}
       volumes:
         - ${{ github.workspace }}:/cre/python
     steps:
@@ -32,8 +31,12 @@ jobs:
 #          ref: main
       - name: cd /cre/python/
         run: (cd /cre/python/)
-      - name: Install spacy textblob_de
-        run: (pip3 install spacy textblob_de)
+      - name: cp mysecrets.py
+        run: (cp mysecrets.orig.py mysecrets.py)
+      - name: Install spacy==2.2.4 textblob_de
+        run: (pip3 install spacy==2.2.4 textblob==0.15.3 textblob_de==0.4.3 nltk)
+      - name: Install geocoder
+        run: (pip3 install geocoder geopandas==0.9)
       - name: Install download de_core_news_md
         run: (python3 -m spacy download de_core_news_md)
       - name: Run entities

.github/workflows/harvest.yml

@@ -3,14 +3,11 @@
 name: harvest:0.0
 
 on:
-  push:
-    branches:
-      - 'main'
   workflow_dispatch:
   schedule:
     # * is a special character in YAML so you have to quote this string : 
 #    daily,every 5 hours #m h d/m m d/w 
-    - cron:  '00 */5 * * *'
+    - cron:  '20 1,6,12,18 * * *'
 #    every 2 hour
 #    - cron:  '10 */2 * * *'
 #    every month 2.nd day
@@ -20,7 +17,7 @@ jobs:
   docker-run-action:
     runs-on: ubuntu-latest
     container:
-      image: tamboraorg/crecoding:2020.0
+      image: credocker/crepython:2024.0
       env:
         NEWSAPI_KEY: ${{ secrets.NEWSAPI_KEY }}
       volumes:
@@ -34,17 +31,19 @@ jobs:
 #        with:
 #          ref: master
       - name: install aiohttp
-        run: (pip3 install aiohttp)
+        run: (uv pip install aiohttp)
       - name: install asyncio
-        run: (pip3 install asyncio)
+        run: (uv pip install asyncio)
+      - name: install deep-translator
+        run: (uv pip install deep-translator)
       - name: cd /cre/python/
         run: (cd /cre/python/)
       - name: cp mysecrets.py
         run: (cp mysecrets.orig.py mysecrets.py)
       - name: Run newsapi
         env:
             NEWSAPI_KEY: ${{ secrets.NEWSAPI_KEY }}
-        run: (python3 newsapi.py)
+        run: (uv run python newsapi.py)
       - name: Submit changes       
         uses: EndBug/add-and-commit@v9
         with:

.github/workflows/heatmap.yml

@@ -0,0 +1,43 @@
+
+# This is a basic workflow to help you get started with Actions
+
+name: heatmap:0.0
+
+on:
+  workflow_dispatch:
+  schedule:
+    # * is a special character in YAML so you have to quote this string : 
+#    weekly #m h d/m m d/w 
+#    - cron:  '30 1 * * 1'
+#    daily #m h d/m m d/w 
+    - cron:  '05 2 * * *'
+
+jobs:
+  docker-run-action:
+    runs-on: ubuntu-latest
+    container:
+      image: credocker/crepython:2024.0
+      volumes:
+        - ${{ github.workspace }}:/cre/python
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set ownership
+        run: |
+          # this is to fix GIT not liking owner of the checkout dir
+          chown -R $(id -u):$(id -g) $PWD
+#        with:
+#          ref: main
+      - name: cd /cre/python/
+        run: (cd /cre/python/)
+      - name: Install shapely cartopy
+        run: (uv pip install shapely cartopy --no-binary shapely --no-binary cartopy)
+      - name: Install folium pillow
+        run: (uv pip install folium pillow)
+      - name: Run diagrams
+        run: (uv run python heatmap.py)
+      - name: Submit changes       
+        uses: EndBug/add-and-commit@v9
+        with:
+#          pull: '--rebase --autostash ...'
+          add: 'img/heatmap.png'
+          tag_push: '--force'

CHECK.md

@@ -0,0 +1,36 @@
+# BASICS  
+### Github Organization  
+:white_check_mark: Github Organization exists  
+:white_check_mark: Github Organization assigned  
+
+---
+
+### NewsAPI  
+:white_check_mark: NewsAPI key exists  
+:white_check_mark: NewsAPI respone fine  
+:white_check_mark: NewsAPI status fine  
+:white_check_mark: NewsAPI results found  
+
+---
+
+### GeonamesAPI  
+:no_entry: Geonames key **missing**:  
+1. Please register with username (i.e.: MomoMusic) at https://www.geonames.org/login  
+2. Assign the choosen username as new organization secret at https://github.com/organizations/MomoMusic/settings/secrets/actions/new  
+   * Name:  **GEONAMES_KEY**   
+   * Value: **Your username here**   
+
+---
+
+### RapidAPI  
+:no_entry: RapidAPI key **missing**:  
+1. Please register at https://rapidapi.com/auth/sign-up  
+2. Login and 'Subscribe to Test' at https://rapidapi.com/bonaipowered/api/google-news22  
+3. Make sure to enter 'Start Free Plan' and press 'Subscribe' - **don't** enter credit card data!  
+2. Copy your API key from (**X-RapidAPI-Key**) from the same site  
+3. Assign the API key as new organization secret at https://github.com/organizations/MomoMusic/settings/secrets/actions/new  
+   * Name:  **RAPIDAPI_KEY**   
+   * Value: **Your key here**   
+
+---
+

HOWTO.md

@@ -28,7 +28,7 @@ The harvester works best with about 20 search-terms (called keyword here) organi
 
 ![image of organisations](images/gh_organisation.png)
 
-* Create a new organization with unique name - i.e. choose your username and append '_news', like 'kmicha_news'
+* Create a new organization with unique name - i.e. choose your username and append '-news', like 'kmicha-news'
 
 ![image of organizations](images/gh_new_org.png)
 
@@ -84,5 +84,25 @@ Attention: You can adapt the keywords later, but for now you should not remove a
 
 ### 7. Remove existing news_20yy_mm.csv files
 
+Inside your repository, goto code, then inside csv folder, select news_2022_mm.csv file (one after the other) and delete it.
+After each deletion,a commit must be done. 
+
+![goto csv](images/gh_csv_folder.png)
+
+![select csv](images/gh_select_news_csv.png)
+
+![delete csv](images/gh_delete_news_csv.png)
+
+![submit csv](images/gh_submit_delete.png)
+
+
 ### 8.) Add the newsapi API-key to your organization
 
+Switch to your organization (the one ending with "-news") and goto the settings tab.
+Scroll down  untill you find the Security section at the left side panel; there open the Secrets-Range and click on Actions.
+
+![new secret](images/gh_secrets_new.png)
+
+Press the "New organization secret" button and add the newsapi-key inside the Value field. For the name use "NEWSAPI_KEY".
+
+![add secret](images/gh_secrets_add.png)

README.md

@@ -29,6 +29,10 @@ To setup you own harvester read the [HOWTO.md](https://github.com/newsWhisperer/
 
 ![DateTopic](img/dates_topics_article_count.png)
 
+# Heatmap
+
+![DateTopic](img/heatmap.png)
+
 
 
 

bayes.py

@@ -8,6 +8,7 @@
 
 
 import nltk
+import sklearn
 from nltk.corpus import stopwords
 from HanTa import HanoverTagger as ht
 from textblob_de import TextBlobDE
@@ -57,6 +58,7 @@ def getNewsDF():
 print(newsDf)   
 
 language = 'ger'
+nltk.download('punkt_tab')
 nltk.download('punkt')
 nltk.download('stopwords')
 tagger = ht.HanoverTagger('morphmodel_'+language+'.pgz')