transform_md.py: fix notes versus history management

.github/workflows/pr.yml

@@ -30,6 +30,15 @@ jobs:
           python-version: '3.11'
           cache: 'poetry'
 
+      - name: Cache pre-commit environment
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pre-commit
+          key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pre-commit-
+
       - name: Run pre-commit checks
         run: |
           set -ex

integration-test.sh

@@ -9,15 +9,17 @@ else
 fi
 
 TOP=$(mktemp -d -p $HOME)
+SRCDIR=$TOP/Notes
+DSTDIR=$TOP/.second-brain
 
-mkdir $TOP/.second-brain $TOP/Notes
+mkdir -p $SRCDIR $DSTDIR
 
 # avoid losing my local env if testing locally :-)
 test ! -f .env
 
 cat > .env <<EOF
-SRCDIR=$TOP/Notes
-DSTDIR=$TOP/.second-brain
+SRCDIR=$SRCDIR
+DSTDIR=$DSTDIR
 EOF
 
 bash -x ./install-systemd-services.sh

monitor.sh

@@ -33,6 +33,9 @@ fi
 
 echo "Finished processing $SRC ($TRANSFORM)"
 
-inotifywait -m -e CLOSE_WRITE,DELETE "$SRC"|while read dir event fname; do echo "${dir}${fname} $event" 1>&2; echo "${dir}${fname}"; done | "$TRANSFORM" "-" "$DST"
+inotifywait -m -e CLOSE_WRITE,DELETE "$SRC"|while read dir event fname; do
+  echo "${dir}${fname} $event" 1>&2
+  echo "${dir}${fname}"
+done | "$TRANSFORM" "-" "$DST"
 
 # monitor.sh ends here

transform_md.py

@@ -394,16 +394,15 @@ def write_output_file(md_file, out_dir, metadata):
     last_accessed_at = datetime.datetime.fromtimestamp(md_stat.st_mtime)
     basename = os.path.basename(md_file[:-3])
     if metadata is None:
-        # add metadata and remove header from content from the first file
         metadata, content = get_metadata(output)
-        metadata["type"] = "notes"
     else:
         new_metadata, content = get_metadata(output)
         metadata.update(new_metadata)
-        metadata["type"] = "history"
     metadata["last_accessed_at"] = last_accessed_at
     if "url" not in metadata:
         metadata["url"] = f"file://{md_file}"
+    if "type" not in metadata:
+        metadata["type"] = "notes"
     print(f"saving {md_file=} with {metadata=}", file=sys.stderr)
     omdname = get_output_file_path(out_dir, basename)
     saved = save_content(
@@ -454,10 +453,8 @@ def process_md_file(fname, out_dir, checksum_store):
         return False
     if checksum_store.has_file_changed(fname) is not False or not os.path.exists(oname):
         md_files = split_md_file(fname, os.path.join(out_dir, "Markdown"))
-        metadata = None
-        # extract the metadata and content from the first file and pass it to the others
         for md_file in md_files:
-            metadata = write_output_file(md_file, out_dir, metadata)
+            write_output_file(md_file, out_dir, None)
     else:
         print(f"skipping {fname} / {oname} as content did not change", file=sys.stderr)
         return False
@@ -471,7 +468,7 @@ def main(in_dir, out_dir):
     # read filenames from stdin
     if in_dir == "-":
         print("Reading filenames from stdin", file=sys.stderr)
-        for fname in sys.stdin:
+        for fname in sys.stdin.readlines():
             process_md_file(fname.rstrip(), out_dir, checksum_store)
     else:
         # scan input dir

transform_txt.py

@@ -131,7 +131,7 @@ def main(in_dir: str, out_dir: str):
     # read filenames from stdin
     if in_dir == "-":
         print("Reading filenames from stdin", file=sys.stderr)
-        for fname in sys.stdin:
+        for fname in sys.stdin.readlines():
             process_file(fname.rstrip(), out_dir, indexer, splitter)
     else:
         # scan input dir