marc-itemextract

# sirsi was inadvertantly modified to accommodate NALIS -- must be modified

knownsystems=("destiny" "eos" "janium" "koha" "lc2" "opals" "sierra" "sirsi" "softlink" "tlc" "virtua" "wms")

if [[ -z $2 ]];then
	echo "Usage: marc-itemextract [filename] [systemtype] "
	echo 
	echo "001 is automatically stored and output."  
	echo "Items are expected to be one item per MARC tag"
	echo
	echo "The following systems are known"
	echo  ${knownsystems[@]}
	exit
else
	infile="${1}"
	system="${2}"
	tags=()
	labels=()

	fileroot=$(echo "${infile}" | sed 's/\.....\?$//')
	outfile="${fileroot}.tsv"
	rm -f ${outfile}
fi

foundsystem=0

for checksystem in "${knownsystems[@]}";do
	if [[ ${system} == $checksystem ]];then foundsystem=1;fi
done


if [[ ${foundsystem} -eq 0 ]];then
	echo "You must specify a known system from the following"
	echo  ${knownsystems[@]}
	exit
fi

## be sure to map call number and location to holdid

if [[ ${system} == "destiny" ]];then
	special[1]="852"

	identifier="001"
	item_field="852p"
	item_call_subfield="h"
	location_subfield="b"

	tags[1]="p"
	tags[2]="h"
	tags[3]="u"
	tags[4]="b"
	tags[5]="j"
	tags[6]="t"
	tags[7]="z"
	tags[8]="r"
	tags[9]="n"
	tags[10]="f"
	tags[11]="9"

	labels[1]="barcode"
	labels[2]="call#"
	labels[3]="itype"
	labels[4]="location"
	labels[5]="volume"
	labels[6]="copy"
	labels[7]="note"
	labels[8]="catdate"
	labels[9]="staffnote"
	labels[10]="collectionnote"
	labels[11]="price"
fi

if [[ ${system} == "eos" ]];then
	identifier="001"
	item_field="949z"
	item_call_subfield="a"
	location_subfield="h"

	tags[1]="z"
	tags[2]="a"
	tags[3]="o"
	tags[4]="b"

	labels[1]="barcode"
	labels[2]="call#"
	labels[3]="itype"
	labels[4]="location"
fi

if [[ ${system} == "janium" ]];then
	identifier="001"
	item_field="999"
	item_call_subfield="a"
	location_subfield="l"

	tags[1]="m"
	tags[2]="l"
	tags[3]="r"
	tags[4]="i"
	tags[5]="m"
	tags[6]="t"
	tags[7]="u"
	tags[8]="v"
	tags[9]="x"
	tags[10]="z"
	tags[11]="0"
	tags[12]="r"
	tags[13]="f"
	tags[14]="g"
	tags[15]="k"

	labels[1]="library"
	labels[2]="location"
	labels[3]="call"
	labels[4]="barcode2"
	labels[5]="barcode"
	labels[6]="mattype"
	labels[7]="staff_note"
	labels[8]="volume"
	labels[9]="staff_note2"
	labels[10]="staff_note3"
	labels[11]="copy"
	labels[12]="loan_type"
	labels[13]="note"
	labels[14]="note_inventory"
	labels[15]="cat_date"
fi

if [[ ${system} == "koha" ]];then
	identifier="001"
	item_field="952p"
	item_call_subfield="o"
	location_subfield="b"

	tags[1]="p"
	tags[2]="8"
	tags[3]="o"
	tags[4]="t"
	tags[5]="y"
	tags[6]="g"
	tags[7]="z"
	tags[8]="d"
	tags[9]="b"
	tags[10]="7"

	labels[1]="barcode"
	labels[2]="call#prefix"
	labels[3]="call#"
	labels[4]="copy"
	labels[5]="mattype"
	labels[6]="price"
	labels[7]="public_note"
	labels[8]="inventory_date"
	labels[9]="location"
	labels[10]="loan_type"
fi

if [[ ${system} == "lc2" ]];then
	identifier="001"
	item_field="949z"
	item_call_subfield="a"
	location_subfield="h"
	bib_mattype_field="380"

	tags[1]="z"
	tags[2]="a"
	tags[3]="q"
	tags[4]="s"
	tags[5]="c"
	tags[6]="v"
	tags[7]="g"
	tags[8]="d"
	tags[9]="p"
	tags[10]="h"
	tags[11]="x"
	tags[12]="i"

	labels[1]="barcode"
	labels[2]="call#"
	labels[3]="call#suffix"
	labels[4]="call#suffix2"
	labels[5]="copy"
	labels[6]="volume"
	labels[7]="price"
	labels[8]="volume_dates"
	labels[9]="part"
	labels[10]="location"
	labels[11]="enum"
	labels[12]="staff_note"
fi

if [[ ${system} == "opals" ]];then
	identifier="001"
	item_field="852p"
	item_call_subfield="h"
	location_subfield="b"

	tags[1]="p"
	tags[2]="k"
	tags[3]="h"
	tags[4]="i"
	tags[5]="m"
	tags[6]="v"
	tags[7]="t"
	tags[8]="3"
	tags[9]="9"

	labels[1]="barcode"
	labels[2]="call#prefix"
	labels[3]="call#"
	labels[4]="call#"
	labels[5]="call#suffix"
	labels[6]="volume"
	labels[7]="copy"
	labels[8]="mattype"
	labels[9]="price"
fi

if [[ ${system}  == "sierra"  ]];then
	bib_call_field="090"
	identifier="907a"
	item_field="945l"
	item_call_subfield="a"
	location_subfield="l"

	tags[1]="a"
	tags[2]="b"
	tags[3]="c"
	tags[4]="g"
	tags[5]="i"
	tags[6]="j"
	tags[7]="l"
	tags[8]="m"
	tags[9]="n"
	tags[10]="o"
	tags[11]="p"
	tags[12]="q"
	tags[13]="r"
	tags[14]="s"
	tags[15]="t"
	tags[16]="u"
	tags[17]="v"
	tags[18]="w"
	tags[19]="x"
	tags[20]="y"
	tags[21]="z"

	labels[1]="call_marc"
	labels[2]="call_nonmarc"
	labels[3]="volume"
	labels[4]="copy"
	labels[5]="barcode"
	labels[6]="agency"
	labels[7]="location"
	labels[8]="msg"
	labels[9]="note"
	labels[10]="icode2"
	labels[11]="price"
	labels[12]="imsg"
	labels[13]="opacmsg"
	labels[14]="status"
	labels[15]="itype"
	labels[16]="totchk"
	labels[17]="totrenwl"
	labels[18]="ytdcirc"
	labels[19]="lycirc"
	labels[20]="record#item"
	labels[21]="created"
fi

if [[ ${system}  == "sirsi"  ]];then
	bib_call_field="092"
	identifier="001"
	item_field="999i"
	item_call_subfield="a"
	location_subfield="m"

	tags[1]="a"
	tags[2]="v"
	tags[3]="w"
	tags[4]="c"
	tags[5]="h"
	tags[6]="i"
	tags[7]="m"
	tags[8]="d"
	tags[9]="e"
	tags[10]="f"
	tags[11]="g"
	tags[12]="j"
	tags[13]="k"
	tags[14]="l"
	tags[15]="n"
	tags[16]="o"
	tags[17]="p"
	tags[18]="q"
	tags[19]="r"
	tags[20]="s"
	tags[21]="t"
	tags[22]="u"
	tags[23]="x"
	tags[24]="z"

	
	labels[1]="call"
	labels[2]="volume"
	labels[3]="callType"
	labels[4]="copy"
	labels[5]="holdingCode"
	labels[6]="barcode"
	labels[7]="library"
	labels[8]="catDate"
	labels[9]="lastActivity"
	labels[10]="lastCharged"
	labels[11]="dateInventoried"
	labels[12]="pieces"
	labels[13]="status"
	labels[14]="location"
	labels[15]="totalCharges"
	labels[16]="note"
	labels[17]="price"
	labels[18]="inHouseCharges"
	labels[19]="circFlag"
	labels[20]="permanentFlag"
	labels[21]="mattype"
	labels[22]="acqDate"
	labels[23]="itemCat1"
	labels[24]="itemCat2"
fi

if [[ ${system}  == "softlink"  ]];then
	identifier="907a"
	item_field="995a"
	item_call_subfield="k"
	location_subfield="a"

	tags[1]="a"
	tags[2]="j"
	tags[3]="y"
	tags[4]="Z"
	tags[5]="h"
	tags[6]="e"
	tags[7]="u"
	tags[8]="J"
	
	labels[1]="location"
	labels[2]="call"
	labels[3]="barcode"
	labels[4]="itype"
	labels[5]="copy"
	labels[6]="accession"
	labels[7]="status"
	labels[8]="note"
fi

if [[ ${system} == "tlc" ]];then
	identifier="001"
	item_field="949g"
	item_call_subfield="a"
	location_subfield="D"
	bib_call_field="090"
	bib_call_fallback="050"
	bib_mattype_field="380"

	tags[1]="a"
	tags[2]="c"
	tags[3]="g"
	tags[4]="n"
	tags[5]="p"
	tags[6]="i"
	tags[7]="j"
	tags[8]="k"
	tags[9]="q"
	tags[10]="7"
	tags[11]="5"

	labels[1]="location"
	labels[2]="call#"
	labels[3]="barcode"
	labels[4]="copy"
	labels[5]="price"
	labels[6]="volume"
	labels[7]="enumeration"
	labels[8]="enumeration_2"
	labels[9]="staff_note"
	labels[10]="staff_note2"
	labels[11]="status"
fi

if [[ ${system} == "virtua" ]];then
	identifier="001"
	item_field="949D"
	item_call_subfield="a"
	location_subfield="D"
	bib_call_field="090"
	bib_call_fallback="050"

	tags[1]="a"
	tags[2]="b"
	tags[3]="6"
	tags[4]="F"
	tags[5]="D"
	tags[6]="x"
	tags[7]="o"
	tags[8]="p"
	tags[9]="q"
	tags[10]="G"

	labels[1]="call#"
	labels[2]="call#cutter"
	labels[3]="barcode"
	labels[4]="copy"
	labels[5]="location"
	labels[6]="itype"
	labels[7]="staff_note"
	labels[8]="public_note"
	labels[9]="checkin_note"
	labels[10]="temp_location"
fi

if [[ ${system} == "wms"  ]];then
	identifier="004"
	item_field="876p"
	item_call_subfield=""
	location_subfield=""

	tags[1]="p"
	tags[2]="3"
	tags[3]="x"
	tags[4]="z"

	labels[1]="barcode"
	labels[2]="volume"
	labels[3]="imsg"
	labels[4]="opacmsg"
fi

taglist="$(echo ${tags[@]})"
labellist="$(echo ${labels[@]})"
speciallist="$(echo ${special[@]})"

echo "Starting processing. Please wait"

read -r -d '' awkscript << "ENDOFAWK"
#!/usr/bin/awk -f 

BEGIN { records_found = 0 

		subfield_regex=SFS"."

      if (length("а") != 2) {
                badawk = 1
                printf("Your version of awk does not support  -- you need a version that supports the -b switch\\n")
                exit
        } else {

		split(TAGLIST, tags, " ")
		split(LABELLIST, labels, " ")
		split(SPECIALLIST, special, " ")

		for (j=1;j<=length(tags);j++) {value=tags[j];tag_index[value]=j}

		num_tags = length(tags)

		required_item_subfield = substr(ITEMFIELD, 4, 1)
		ITEMFIELD = substr(ITEMFIELD, 1, 3)
	}
}

function destiny_852() { 
	# material type
	sub("@a", SFS"u", field_content)
	vol_subfield=""

	# cat date
	sub("@c", SFS"r", field_content)

	# collection 
	sub("@f", SFS"f", field_content)

	# remap first $x to item note $n 
	sub(SFS"x", SFS"n", field_content)

	# volume info is in @j@j@j@i@i@i with label in j
	# and enum in @i. Pair things up and string it together

	match(field_content, "@j[^"SFS"]*", val)
	split(val[0], volume_info, "@")

	for (i in volume_info) {
		sub(/^[ij]/, "", volume_info[i])
	}
	vol_subfield = volume_info[2]" "volume_info[5]" "volume_info[3]" "volume_info[6]" "volume_info[4]" "volume_info[7]
	sub(" *$", "", vol_subfield)
	field_content = field_content""SFS"j"vol_subfield

	# pitch all subfield b's
	sub("@b", SFS"8", field_content)

	# pitch all remaining @fields
	gsub("@[^"SFS"]*", "", field_content)
}

function extract_subfields() { 
	real_item = 0
	for (j=1;j<=num_tags;j++) { item_subfield[j] = "" }

	split(field_content, subfields, SFS)
	
	for (subfield=1;subfield<=length(subfields);subfield++) {
		extracted_tag=substr(subfields[subfield], 1, 1)
		index_position=tag_index[extracted_tag]

		if (index_position > 0) {
		if (substr(subfields[subfield], 1, 1) == extracted_tag) {
			if (extracted_tag == required_item_subfield) {real_item = 1}
			if (item_subfield[index_position] == "") {
				item_subfield[index_position] = substr(subfields[subfield],2)
				gsub(/^[ \\-]*$/, "", item_subfield[index_position])
			} else { 
				item_subfield[index_position] = item_subfield[index_position]";"substr(subfields[subfield],2) 
			}

			gsub(/^ *+| +$/, "", item_subfield[index_position])

			if (tags[index_position] == LOCATION) { location = item_subfield[index_position] }
			if (tags[index_position] == CALL_PREFIX) { callprefix = item_subfield[index_position] }
			if (tags[index_position] == CALL) { callnum = item_subfield[index_position] }
			if (tags[index_position] == CALL_SUFFIX) { callsuffix = item_subfield[index_position] }
		}	
	}
	}

	if (IDENTIFIER == "004") {
		holdid = tag001 
		gsub(/[^a-zA-Z0-9]/, "", bibno)
	} else {
		holdid =  bibno""location""callprefix""callnum""callsuffix
	}

	gsub(/[^a-zA-Z0-9]/, "", holdid)

	lineout = tag001"\t"bibno"\t"holdid"\t"bib_call"\t"holdings_type"\t"bib_mattype

	for (j=1;j<=num_tags;j++) {
		lineout = lineout"\t"item_subfield[j]
	}
	
	if (bibno != "" && real_item == 1) { print lineout >> OUTFILE;numItems++ }
	lineout = ""
}

function extract_identifiers() {

	tag001=tag004=bib_subtype=bibno=bib_call=holdid=callnum=location=""
	tagid = substr(IDENTIFIER, 4, 1)

	for (i=1; i<=directory_length; i=i+12) {
		if (substr(directory, i, 3) == "001") {

			field_length = substr(directory, i + 3, 4) + 0
			starting_pos = substr(directory, i + 7, 5) 

			tag001 = substr(record_content, starting_pos, field_length)
			gsub(/^ *+| +$/, "", tag001)
			gsub(FS, "", tag001)
		}
		if (substr(directory, i, 3) == "004") {

			field_length = substr(directory, i + 3, 4) + 0
			starting_pos = substr(directory, i + 7, 5) 

			tag004 = substr(record_content, starting_pos, field_length - 1)
			gsub(/^ *+| +$/, "", tag004)
		}
		if (substr(directory, i, 3) == "008") {

			field_length = substr(directory, i + 3, 4) + 0
			starting_pos = substr(directory, i + 7, 5) 

			bib_subtype = substr(record_content, starting_pos + 24, 1)
			gsub(/[^a-z]/, "", bib_subtype)
			bib_mattype = bib_mattype""bib_subtype
		}

		if (bibno == "") {
			if (substr(directory, i, 3) == substr(IDENTIFIER, 1, 3)) {

				field_length = substr(directory, i + 3, 4) + 0
				starting_pos = substr(directory, i + 7, 5) 

				split(field_content, subfields, SFS)

				bibno = substr(record_content, starting_pos, field_length)
				split(bibno, subfields, SFS)
	
				for (subfield in subfields) {
					if (tagid == substr(subfields[subfield], 1, 1)) {
						bibno = substr(subfields[subfield],2)
					}	
				}
			#gsub(/^ *+| +$/, "", bibno)
			gsub(/[^a-zA-Z0-9]/, "", bibno)
			}
		}
	}
}

function extract_items() {
	for (d_iter=1; d_iter<=directory_length; d_iter=d_iter+12) {
		marc_tag = substr(directory, d_iter, 3)
		####
      if (marc_tag == BIB_CALL_FIELD) {

         field_length = substr(directory, d_iter + 3, 4) + 0
         starting_pos = substr(directory, d_iter + 7, 5)

         bib_call = substr(record_content, starting_pos + 5, field_length - 5)
         gsub(subfield_regex, " ", bib_call)
         }

      if (marc_tag == BIB_CALL_FALLBACK) {
			if (bib_call == "") {
         	field_length = substr(directory, d_iter + 3, 4) + 0
         	starting_pos = substr(directory, d_iter + 7, 5)

         	bib_call = substr(record_content, starting_pos + 5, field_length - 5)
         	gsub(subfield_regex, " ", bib_call)
				}
         }

      if (marc_tag == BIB_MATTYPE_FIELD) {

         field_length = substr(directory, d_iter + 3, 4) + 0
         starting_pos = substr(directory, d_iter + 7, 5)

         bib_mattype = substr(record_content, starting_pos + 5, field_length - 5)
         gsub(subfield_regex, " ", bib_mattype)
         }

		if (marc_tag == ITEMFIELD) {

			field_length = substr(directory, d_iter + 3, 4) + 0
			starting_pos = substr(directory, d_iter + 7, 5) 
			field_content = ""

			if (substr(record_content, starting_pos + 3, 1) == SFS) {
				field_content = substr(record_content, starting_pos + 3, field_length - 3)
			} else {
				field_content = substr(record_content, starting_pos + 1, field_length - 1)
			}

			for (special_iter in special) { 
				if (marc_tag == special[special_iter]) { 
					functionByVariable=SYSTEM"_"marc_tag

					@functionByVariable()		
				} 
			}

			extract_subfields()
		}
	}
}

{

leader=substr($0,1,24)
baseaddress=substr(leader,13, 5) + 0
directory=substr($0,25, baseaddress - 25)
directory_length=length(directory) 
directory_check=(directory_length % 12)
record_content=substr($0, baseaddress + 1)
bib_mattype=substr(leader, 7, 1)
holdings_type=substr(leader, 8, 1)

if (directory_check == 0) {
	if (NR == 1) { 
		lineout = "tag001\tbib_no\tholdid\tbib_call\trectype\tbib_mattype"
		for (i=1;i<=num_tags;i++) {
			lineout = lineout"\t"labels[i]
		}
		print lineout > OUTFILE
		lineout = ""
	}
	extract_identifiers()
	extract_items()
}

if (NR % 1000 == 0){ printf "Records processed: "NR"\r" }

}
END { print NR" MARC records have been processed. "numItems" item records were sent to "OUTFILE }

ENDOFAWK

echo -e "${awkscript}" > tmp_checkmarc
chmod 700 tmp_checkmarc

awk -v RS=$'\x1d' -v ORS="\n" -v SFS=$'\x1f' -v FS=$'\x1e' -v OFS="\t" -v IDENTIFIER="${identifier}" -v ITEMFIELD="${item_field}" -v CALL="${item_call_subfield}" -v CALL_PREFIX="${item_call_prefix}" -v CALL_SUFFIX="${item_call_suffix}" -v BIB_CALL_FIELD="${bib_call_field}" -v BIB_CALL_FALLBACK="${bib_call_fallback}" -v BIB_MATTYPE_FIELD="${bib_mattype_field}" -v LOCATION="${location_subfield}" -v TAGLIST="${taglist}" -v LABELLIST="${labellist}" -v SPECIALLIST="${speciallist}" -v SYSTEM="${system}" -v OUTFILE="${outfile}" -b -f tmp_checkmarc "${infile}"
echo

rm -f tmp_checkmarc