feat(#162): final recipes for composing found metadata into final CSV

.github/workflows/collect.yml

@@ -75,16 +75,20 @@ jobs:
             just extract "../after-swc.csv" "../after-extract.csv"
             just maven "../after-extract.csv" "$TOKEN" \
               "../after-maven.csv"
+            just lens "../after-maven.csv" "../after-lens.csv"
+            just final "../after-lens.csv" "../final.csv"
           } 2>&1 | tee -a collect.log
           sed -i "s|$TOKEN|REDACTED|g" collect.log
-          cp "repos.csv" "collection/${{ inputs.out }}"
-          cp "repos-with-pulls.csv" "collection/${{ inputs.out }}"
-          cp "after-filter.csv" "collection/${{ inputs.out }}"
-          cp "after-mcw.csv" "collection/${{ inputs.out }}"
-          cp "after-swc.csv" "collection/${{ inputs.out }}"
-          cp "after-extract.csv" "collection/${{ inputs.out }}"
-          cp "after-maven.csv" "collection/${{ inputs.out }}"
-          cp "collect.log" collection/${{ inputs.out }}
+          cp "repos.csv" "collection/${{ inputs.out }}/steps"
+          cp "repos-with-pulls.csv" "collection/${{ inputs.out }}/steps"
+          cp "after-filter.csv" "collection/${{ inputs.out }}/steps"
+          cp "after-mcw.csv" "collection/${{ inputs.out }}/steps"
+          cp "after-swc.csv" "collection/${{ inputs.out }}/steps"
+          cp "after-extract.csv" "collection/${{ inputs.out }}/steps"
+          cp "after-maven.csv" "collection/${{ inputs.out }}/steps"
+          cp "after-lens.csv" "collection/${{ inputs.out }}/steps"
+          cp "collect.log" "collection/${{ inputs.out }}"
+          cp "final.csv" "collection/${{ inputs.out }}"
       - uses: JamesIves/github-pages-deploy-action@v4.6.8
         with:
           branch: collect

justfile

@@ -104,6 +104,10 @@ swc repos out="experiment/after-swc.csv" config="resources/swc-words.txt":
 lens repos out="experiment/after-lens.csv":
   cd sr-data && poetry poe lens --repos {{repos}} --out {{out}}
 
+# Compose all found metadata into final CSV.
+final latest out="experiment/final.csv":
+  cd sr-data && poetry poe final --latest {{latest}} --out {{out}}
+
 # Create embeddings.
 embed repos prefix="experiment/embeddings":
   cd sr-data && poetry poe embed --repos {{repos}} --prefix {{prefix}} \

sr-data/pyproject.toml

@@ -105,6 +105,10 @@ args = [{name = "repos"}, {name = "out"}]
 script = "sr_data.steps.lens:main(repos, out)"
 args = [{name = "repos"}, {name = "out"}]
 
+[tool.poe.tasks.final]
+script = "sr_data.steps.final:main(latest, out)"
+args = [{name = "latest"}, {name = "out"}]
+
 [build-system]
 requires = ["setuptools", "wheel"]
 build-backend = "setuptools.build_meta"

sr-data/src/sr_data/steps/final.py

@@ -0,0 +1,52 @@
+"""
+Compose all found metadata into final CSV.
+"""
+# The MIT License (MIT)
+#
+# Copyright (c) 2024 Aliaksei Bialiauski
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import pandas as pd
+
+
+def main(latest, out):
+    frame = pd.read_csv(latest)
+    frame = frame[
+        [
+            "repo",
+            "releases",
+            "issues",
+            "branches",
+            "pulls",
+            "projects",
+            "plugins",
+            "pwars",
+            "pjars",
+            "ppoms",
+            "mcw",
+            "example_wc",
+            "sample_wc",
+            "demonstration_wc",
+            "rlen",
+            "avg_slen",
+            "avg_wlen",
+            "hnum"
+        ]
+    ]
+    frame.to_csv(out, index=False)

sr-data/src/tests/test_final.py

@@ -0,0 +1,48 @@
+"""
+Tests for final.
+"""
+# The MIT License (MIT)
+#
+# Copyright (c) 2024 Aliaksei Bialiauski
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import os.path
+import unittest
+from tempfile import TemporaryDirectory
+
+import pandas as pd
+import pytest
+from sr_data.steps.final import main
+
+
+class TestFinal(unittest.TestCase):
+
+    @pytest.mark.fast
+    def test_composes_into_final_csv(self):
+        with TemporaryDirectory() as temp:
+            path = os.path.join(temp, "final.csv")
+            main(
+                os.path.join(
+                    os.path.dirname(os.path.realpath(__file__)), "to-final.csv"
+                ),
+                path
+            )
+            final = pd.read_csv(path)
+            self.assertEqual(len(final.columns), 18)
+            self.assertEqual(len(final), 1)

sr-data/src/tests/to-final.csv

@@ -0,0 +1,2 @@
+repo,branch,readme,releases,issues,branches,pulls,headings,top,projects,plugins,pwars,pjars,ppoms,hnum,rlen,avg_slen,avg_wlen,mcw,example_wc,sample_wc,demonstration_wc
+foo/bar,master,# fake,1,2,3,4,"['fake']","['fake']",2,[org.springframework.boot:spring-boot-maven-plugin],0,1,1,1,2,1,2.2,"['fake']",1,2,3