Generate topic list #1031
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate topic list | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| jobs: | |
| update-files: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| - uses: actions/checkout@v2 | |
| - name: Install packages | |
| run: sudo apt-get install perl cpanminus curl; sudo snap install yq | |
| - name: Install required CPAN modules | |
| run: sudo cpanm HTML::Parser XML::XPath | |
| - name: Download feed.xml | |
| run: curl -s https://minkorrekt.podigee.io/feed/mp3 -o feed.xml | |
| - name: Pre-process feed.xml | |
| run: | | |
| xpath -n -q -e 'rss/channel/item/title | rss/channel/item/content:encoded/text() | rss/channel/item/pubDate | rss/channel/item/link | rss/channel/item/itunes:episode' feed.xml | \ | |
| perl -C -MHTML::Entities -pe 'decode_entities($_);decode_entities($_);' > preprocessed.txt | |
| #- name: Dump preprocessed.txt | |
| # run: cat preprocessed.txt | |
| - name: Generate minkorrekt-themen.unfiltered.yaml | |
| run: | | |
| cat preprocessed.txt | \ | |
| perl -C -pe 's/\"/\\\"/g;s/<title>((?:Mi|Minkorrekt Folge )([0-9]+)?[^<]+?)<\/title>/- title: "$1"\n episode: "$2"/g;s/<(pubDate|link)>([^<]+)<\/\1>/ $1: "$2"/g;s/<(itunes:episode)>([^<]+)<\/\1>/ itunesEpisode: "$2"/g;s/<[^>]+>//g;s/^\h*Thema ([1-4]):?\h*(?:\\\"|[\x{201C}\x{201D}\x{201E}])(.+?)(?:\\\".*$|\h*?[\x{201C}\x{201D}\x{201E}].*$|\h\x{2013}[^\x{201C}\x{201D}\x{201E}\v]*$)/ topic$1: "$2"/;s/^\h*?Chinagadget der Woche:\h?(.*?)\h*$/ gadget: "$1"/;s/^(?!- title:| topic[1-4]:| pubDate:| link:| gadget:| itunesEpisode:| episode:).*?$//gi;s/^\s*$//' | \ | |
| yq e '[.[]]' - > minkorrekt-themen.unfiltered.yaml | |
| #- name: Dump minkorrekt-themen.unfiltered.yaml | |
| # run: cat minkorrekt-themen.unfiltered.yaml | |
| - name: Generate minkorrekt-themen.yaml | |
| run: yq e '[.[] | select(has("topic1") // has("topic2") // has("topic3") // has("topic4"))]' minkorrekt-themen.unfiltered.yaml > minkorrekt-themen.yaml | |
| #- name: Dump minkorrekt-themen.yaml | |
| # run: cat minkorrekt-themen.yaml | |
| - name: Generate minkorrekt-themen.full.yaml | |
| run: | | |
| cat preprocessed.txt | \ | |
| perl -C -pe 's/\"/\\\"/g;s/<title>((?:Mi|Minkorrekt Folge )([0-9]+)?[^<]+?)<\/title>/- title: "$1"\n episode: "$2"/g;s/<(pubDate|link)>([^<]+)<\/\1>/ $1: "$2"/g;s/<(itunes:episode)>([^<]+)<\/\1>/ itunesEpisode: "$2"/g;s/<[^>]+>//g;s/^\h*Thema ([1-4]):?\h*(.*$)/ topic$1: "$2"/;s/^\h*?Chinagadget der Woche:\h?(.*?)\h*$/ gadget: "$1"/;s/^(?!- title:| topic[1-4]:| pubDate:| link:| gadget:| itunesEpisode:).*?$//gi;s/^\s*$//' | \ | |
| yq e '[.[]]' - > minkorrekt-themen.full.yaml | |
| #- name: Dump minkorrekt-themen.full.yaml | |
| # run: cat minkorrekt-themen.full.yaml | |
| - name: Generate JSON output | |
| run: | | |
| yq eval -o=j minkorrekt-themen.unfiltered.yaml > minkorrekt-themen.unfiltered.json | |
| yq eval -o=j minkorrekt-themen.yaml > minkorrekt-themen.json | |
| yq eval -o=j minkorrekt-themen.full.yaml > minkorrekt-themen.full.json | |
| - name: Generate HTML page | |
| run: | | |
| sed -e '/::minkorrekt-themen.full.json::/ {' -e 'r minkorrekt-themen.full.json' -e 'd' -e '}' index.tpl.html > index.html | |
| - name: Cleanup | |
| run: rm feed.xml preprocessed.txt | |
| - name: Commit updated files | |
| continue-on-error: true | |
| run: | | |
| git config --global user.name 'M!Topics Action' | |
| git config --global user.email 'mtopics-action@fenrikur.de' | |
| git add *.yaml *.json *.html | |
| git commit -m "Updated files from feed" | |
| git push |