99.reduce-unclean

#!/bin/bash
set -euo pipefail
lang=$1
output_file=$2
shift 2

TMPSFX=${JOB_ID:-$$}

mkdir -p $(dirname $output_file)

cat $@ \
| while read batch; do
	find $batch -maxdepth 1 -type f -regex '.*/aligned-[0-9]*.gz$' | while read match; do
		echo $match 1>&2
		paste <(gzip -cd ${match} \
				| awk -F '\t' '{ print 0.0 "\t" $1 "\t"  $2}' \
				| docjoin \
					-r ${batch}/../../../${TARGET_LANG}/$(basename $(dirname $batch))/$(echo $match | sed 's/.*-\([0-9]*\)\.gz/\1/')/url.gz \
					-l $(dirname ${match})/url.gz) \
			  <(gzip -cd $match | cut -f4) \
			  <(gzip -cd $match | cut -f3) \
			  <(gzip -cd $match | cut -f5-)
	done
done \
| pigz -c > $output_file.$TMPSFX
mv $output_file.$TMPSFX $output_file