searchindex.js

Search.setIndex({envversion:46,filenames:["index","sphinx-docs/dblogger","sphinx-docs/kvlayer","sphinx-docs/overview","sphinx-docs/rejester","sphinx-docs/streamcorpus","sphinx-docs/streamcorpus_pipeline","sphinx-docs/yakonfig"],objects:{"":{"--app-name":[4,0,1,"cmdoption-rejester--app-name"],"--begin":[1,0,1,"cmdoption-dblogger--begin"],"--config":[7,0,1,"cmdoption-yakonfig--config"],"--debug":[1,0,1,"cmdoption-dblogger--debug"],"--directories":[6,0,1,"cmdoption-streamcorpus_directory--directories"],"--dump-config":[7,0,1,"cmdoption-yakonfig--dump-config"],"--end":[1,0,1,"cmdoption-dblogger--end"],"--file-lists":[6,0,1,"cmdoption-streamcorpus_directory--file-lists"],"--files":[6,0,1,"cmdoption-streamcorpus_directory--files"],"--in-glob":[6,0,1,"cmdoption-streamcorpus_pipeline--in-glob"],"--input":[6,0,1,"cmdoption-streamcorpus_pipeline--input"],"--logpath":[4,0,1,"cmdoption-rejester--logpath"],"--namespace":[4,0,1,"cmdoption-rejester--namespace"],"--nice":[6,0,1,"cmdoption-streamcorpus_directory--nice"],"--pidfile":[4,0,1,"cmdoption-rejester--pidfile"],"--quiet":[1,0,1,"cmdoption-dblogger--quiet"],"--registry-address":[4,0,1,"cmdoption-rejester--registry-address"],"--rejester":[6,0,1,"cmdoption-streamcorpus_directory--rejester"],"--standalone":[6,0,1,"cmdoption-streamcorpus_directory--standalone"],"--verbose":[1,0,1,"cmdoption-dblogger--verbose"],"--work-spec":[6,0,1,"cmdoption-streamcorpus_directory--work-spec"],"-c":[7,0,1,"cmdoption-yakonfig-c"],"-i":[6,0,1,"cmdoption-streamcorpus_pipeline-i"],"-q":[1,0,1,"cmdoption-dblogger-q"],"-v":[1,0,1,"cmdoption-dblogger-v"],dblogger:[1,1,0,"-"],kvlayer:[2,1,0,"-"],rejester:[4,1,0,"-"],streamcorpus_pipeline:[6,1,0,"-"],yakonfig:[7,1,0,"-"]},"dblogger.DatabaseLogHandler":{"__init__":[1,3,1,""]},"kvlayer._abstract_storage":{AbstractStorage:[2,2,1,""]},"kvlayer._abstract_storage.AbstractStorage":{"delete":[2,3,1,""],check_put_key_value:[2,3,1,""],clear_table:[2,3,1,""],close:[2,3,1,""],delete_namespace:[2,3,1,""],get:[2,3,1,""],increment:[2,3,1,""],log_delete:[2,3,1,""],log_get:[2,3,1,""],log_put:[2,3,1,""],log_scan:[2,3,1,""],log_scan_keys:[2,3,1,""],put:[2,3,1,""],scan:[2,3,1,""],scan_keys:[2,3,1,""],setup_namespace:[2,3,1,""],str_to_value:[2,3,1,""],value_to_str:[2,3,1,""]},"rejester.Registry":{"delete":[4,3,1,""],direct_call:[4,3,1,""],filter:[4,3,1,""],force_clear_lock:[4,3,1,""],get:[4,3,1,""],getitem_reset:[4,3,1,""],len:[4,3,1,""],lock:[4,3,1,""],move:[4,3,1,""],move_all:[4,3,1,""],popitem:[4,3,1,""],popitem_move:[4,3,1,""],popmany:[4,3,1,""],pull:[4,3,1,""],re_acquire_lock:[4,3,1,""],read_lock:[4,3,1,""],reset_priorities:[4,3,1,""],set:[4,3,1,""],set_1to1:[4,3,1,""],update:[4,3,1,""]},"rejester.RejesterQueue":{add_item:[4,3,1,""],check_out_item:[4,3,1,""],dump_queue:[4,3,1,""],renew_item:[4,3,1,""],reserve_items:[4,3,1,""],return_item:[4,3,1,""],worker_id:[4,4,1,""]},"rejester.TaskMaster":{"__init__":[4,3,1,""],AVAILABLE:[4,4,1,""],BLOCKED:[4,4,1,""],FAILED:[4,4,1,""],FINISHED:[4,4,1,""],IDLE:[4,4,1,""],PENDING:[4,4,1,""],RUN:[4,4,1,""],TERMINATE:[4,4,1,""],add_dependent_work_units:[4,3,1,""],add_work_units:[4,3,1,""],clear:[4,3,1,""],config:[4,4,1,""],default_lifetime:[4,4,1,""],del_work_units:[4,3,1,""],dump:[4,3,1,""],enough_memory:[4,4,1,""],get_assigned_work_unit:[4,3,1,""],get_child_work_units:[4,3,1,""],get_heartbeat:[4,3,1,""],get_mode:[4,3,1,""],get_work:[4,3,1,""],get_work_spec:[4,3,1,""],get_work_unit_status:[4,3,1,""],get_work_units:[4,3,1,""],idle_all_workers:[4,3,1,""],inspect_work_unit:[4,3,1,""],iter_work_specs:[4,3,1,""],list_available_work_units:[4,3,1,""],list_blocked_work_units:[4,3,1,""],list_failed_work_units:[4,3,1,""],list_finished_work_units:[4,3,1,""],list_pending_work_units:[4,3,1,""],list_work_specs:[4,3,1,""],list_work_units:[4,3,1,""],mode_counts:[4,3,1,""],nice:[4,3,1,""],num_available:[4,3,1,""],num_blocked:[4,3,1,""],num_failed:[4,3,1,""],num_finished:[4,3,1,""],num_pending:[4,3,1,""],num_tasks:[4,3,1,""],registry:[4,4,1,""],remove_available_work_units:[4,3,1,""],remove_blocked_work_units:[4,3,1,""],remove_failed_work_units:[4,3,1,""],remove_finished_work_units:[4,3,1,""],remove_pending_work_units:[4,3,1,""],reset_all:[4,3,1,""],retry:[4,3,1,""],set_mode:[4,3,1,""],set_work_spec:[4,3,1,""],status:[4,3,1,""],update_bundle:[4,3,1,""],validate_work_spec:[4,5,1,""],worker_heartbeat:[4,3,1,""],worker_id:[4,4,1,""],worker_register:[4,3,1,""],worker_unregister:[4,3,1,""],workers:[4,3,1,""]},"rejester.WorkUnit":{"__init__":[4,3,1,""],default_lifetime:[4,4,1,""],expires:[4,4,1,""],fail:[4,3,1,""],failed:[4,4,1,""],finish:[4,3,1,""],finished:[4,4,1,""],key:[4,4,1,""],module:[4,4,1,""],registry:[4,4,1,""],run:[4,3,1,""],spec:[4,4,1,""],terminate:[4,3,1,""],update:[4,3,1,""],work_spec_name:[4,4,1,""],worker_id:[4,4,1,""]},"rejester.Worker":{config:[4,4,1,""],environment:[4,3,1,""],heartbeat:[4,3,1,""],lifetime:[4,4,1,""],parent:[4,4,1,""],register:[4,3,1,""],run:[4,3,1,""],task_master:[4,4,1,""],unregister:[4,3,1,""],worker_id:[4,4,1,""]},"rejester.exceptions":{EnvironmentError:[4,7,1,""],ItemInUseError:[4,7,1,""],LockError:[4,7,1,""],LostLease:[4,7,1,""],NoSuchWorkSpecError:[4,7,1,""],NoSuchWorkUnitError:[4,7,1,""],PriorityRangeEmpty:[4,7,1,""],ProgrammerError:[4,7,1,""],RejesterException:[4,7,1,""]},"rejester.exceptions.NoSuchWorkSpecError":{work_spec_name:[4,4,1,""]},"rejester.exceptions.NoSuchWorkUnitError":{work_unit_name:[4,4,1,""]},"rejester.workers":{ForkWorker:[4,2,1,""],HeadlessWorker:[4,2,1,""],MultiWorker:[4,2,1,""],SingleWorker:[4,2,1,""],run_worker:[4,6,1,""]},"rejester.workers.ForkWorker":{check_spinning_children:[4,3,1,""],clear_signal_handlers:[4,3,1,""],debug:[4,3,1,""],do_some_work:[4,3,1,""],live_log_child:[4,3,1,""],log:[4,3,1,""],log_spewer:[4,3,1,""],run:[4,3,1,""],scram:[4,3,1,""],set_signal_handlers:[4,3,1,""],start_log_child:[4,3,1,""],stop_all_children:[4,3,1,""],stop_gracefully:[4,3,1,""],stop_log_child:[4,3,1,""]},"rejester.workers.MultiWorker":{run:[4,3,1,""]},"rejester.workers.SingleWorker":{EXIT_BORED:[4,4,1,""],EXIT_EXCEPTION:[4,4,1,""],EXIT_SUCCESS:[4,4,1,""],as_child:[4,5,1,""],run:[4,3,1,""],run_one:[4,3,1,""]},"streamcorpus.Annotator":{annotation_time:[5,4,1,""],annotator_id:[5,4,1,""]},"streamcorpus.Attribute":{attribute_type:[5,4,1,""],evidence:[5,4,1,""],mention_id:[5,4,1,""],sentence_id:[5,4,1,""],value:[5,4,1,""]},"streamcorpus.AttributeType":{ORG_ALTERNATE_NAMES:[5,4,1,""],ORG_NUMBER_OF_EMPLOYEES_MEMBERS:[5,4,1,""],PER_AGE:[5,4,1,""],PER_ALTERNATE_NAMES:[5,4,1,""],PER_CAUSE_OF_DEATH:[5,4,1,""],PER_CHARGES:[5,4,1,""],PER_GENDER:[5,4,1,""],PER_TITLE:[5,4,1,""]},"streamcorpus.Chunk":{close:[5,3,1,""],flush:[5,3,1,""],read_msg_impl:[5,3,1,""],write_msg_impl:[5,3,1,""]},"streamcorpus.ContentItem":{attributes:[5,4,1,""],clean_html:[5,4,1,""],clean_visible:[5,4,1,""],encoding:[5,4,1,""],external_ids:[5,4,1,""],labels:[5,4,1,""],language:[5,4,1,""],logs:[5,4,1,""],media_type:[5,4,1,""],raw:[5,4,1,""],relations:[5,4,1,""],sentence_blobs:[5,4,1,""],sentences:[5,4,1,""],taggings:[5,4,1,""]},"streamcorpus.EntityType":{CUSTOM_TYPE:[5,4,1,""],DATE:[5,4,1,""],EVENT:[5,4,1,""],FAC:[5,4,1,""],GPE:[5,4,1,""],LIST:[5,4,1,""],LOC:[5,4,1,""],MISC:[5,4,1,""],MONEY:[5,4,1,""],NATIONALITY:[5,4,1,""],ORG:[5,4,1,""],PER:[5,4,1,""],PERCENT:[5,4,1,""],RELIGION:[5,4,1,""],TIME:[5,4,1,""],TITLE:[5,4,1,""],URL:[5,4,1,""],VEH:[5,4,1,""],WEA:[5,4,1,""],email:[5,4,1,""],phone:[5,4,1,""]},"streamcorpus.FlagType":{PROFILE:[5,4,1,""]},"streamcorpus.Gender":{FEMALE:[5,4,1,""],MALE:[5,4,1,""]},"streamcorpus.Label":{annotator:[5,4,1,""],comments:[5,4,1,""],flags:[5,4,1,""],mentions:[5,4,1,""],offsets:[5,4,1,""],positive:[5,4,1,""],relevance:[5,4,1,""],stream_id:[5,4,1,""],target:[5,4,1,""]},"streamcorpus.Language":{code:[5,4,1,""],name:[5,4,1,""]},"streamcorpus.MentionType":{NAME:[5,4,1,""],NOM:[5,4,1,""],PRO:[5,4,1,""]},"streamcorpus.Offset":{content_form:[5,4,1,""],first:[5,4,1,""],length:[5,4,1,""],type:[5,4,1,""],value:[5,4,1,""],xpath:[5,4,1,""]},"streamcorpus.OffsetType":{BYTES:[5,4,1,""],CHARS:[5,4,1,""],LINES:[5,4,1,""]},"streamcorpus.Rating":{annotator:[5,4,1,""],comments:[5,4,1,""],contains_mention:[5,4,1,""],flags:[5,4,1,""],mentions:[5,4,1,""],relevance:[5,4,1,""],target:[5,4,1,""]},"streamcorpus.Relation":{mention_id_1:[5,4,1,""],mention_id_2:[5,4,1,""],relation_type:[5,4,1,""],sentence_id_1:[5,4,1,""],sentence_id_2:[5,4,1,""]},"streamcorpus.RelationType":{ART_UserOwnerInventorManufacturer:[5,4,1,""],Business_DeclareBankruptcy:[5,4,1,""],Business_EndOrg:[5,4,1,""],Business_MergeOrg:[5,4,1,""],Business_StartOrg:[5,4,1,""],Conflict_Attack:[5,4,1,""],Conflict_Demonstrate:[5,4,1,""],Contact_Meet:[5,4,1,""],Contact_PhoneWrite:[5,4,1,""],FAC_LOCATED:[5,4,1,""],FAC_OWNER:[5,4,1,""],FAC_VISITED_BY:[5,4,1,""],GENAFF_CitizenResidentReligionEthnicity:[5,4,1,""],GENAFF_OrgLocation:[5,4,1,""],Justice_Acquit:[5,4,1,""],Justice_Appeal:[5,4,1,""],Justice_ArrestJail:[5,4,1,""],Justice_ChargeIndict:[5,4,1,""],Justice_Convict:[5,4,1,""],Justice_Execute:[5,4,1,""],Justice_Extradite:[5,4,1,""],Justice_Fine:[5,4,1,""],Justice_Pardon:[5,4,1,""],Justice_ReleaseParole:[5,4,1,""],Justice_Sentence:[5,4,1,""],Justice_Sue:[5,4,1,""],Justice_TrialHearing:[5,4,1,""],Life_BeBorn:[5,4,1,""],Life_Die:[5,4,1,""],Life_Divorce:[5,4,1,""],Life_Injure:[5,4,1,""],Life_Marry:[5,4,1,""],Movement_Transport:[5,4,1,""],ORGAFF_Employment:[5,4,1,""],ORGAFF_Founder:[5,4,1,""],ORGAFF_InvestorShareholder:[5,4,1,""],ORGAFF_Membership:[5,4,1,""],ORGAFF_Ownership:[5,4,1,""],ORGAFF_SportsAffiliation:[5,4,1,""],ORGAFF_StudentAlum:[5,4,1,""],ORG_ATTENDED:[5,4,1,""],ORG_CITY_OF_HEADQUARTERS:[5,4,1,""],ORG_COUNTRY_OF_HEADQUARTERS:[5,4,1,""],ORG_DATE_DISSOLVED:[5,4,1,""],ORG_DATE_FOUNDED:[5,4,1,""],ORG_FOUNDED_BY:[5,4,1,""],ORG_MEMBERS:[5,4,1,""],ORG_MEMBER_OF:[5,4,1,""],ORG_PARENTS:[5,4,1,""],ORG_POLITICAL_OR_RELIGIOUS_AFFILIATION:[5,4,1,""],ORG_SHAREHOLDERS:[5,4,1,""],ORG_STATEORPROVINCE_OF_HEADQUARTERS:[5,4,1,""],ORG_SUBSIDIARIES:[5,4,1,""],ORG_TOP_MEMBERS_EMPLOYEES:[5,4,1,""],ORG_VISITED:[5,4,1,""],ORG_WEBSITE:[5,4,1,""],PARTWHOLE_Artifact:[5,4,1,""],PARTWHOLE_Geographical:[5,4,1,""],PARTWHOLE_Subsidiary:[5,4,1,""],PERSOC_Business:[5,4,1,""],PERSOC_Family:[5,4,1,""],PERSOC_LastingPersonal:[5,4,1,""],PER_ATTENDED:[5,4,1,""],PER_CHILDREN:[5,4,1,""],PER_CITIES_OF_RESIDENCE:[5,4,1,""],PER_CITY_OF_BIRTH:[5,4,1,""],PER_CITY_OF_DEATH:[5,4,1,""],PER_COUNTRIES_OF_RESIDENCE:[5,4,1,""],PER_COUNTRY_OF_BIRTH:[5,4,1,""],PER_COUNTRY_OF_DEATH:[5,4,1,""],PER_DATE_OF_BIRTH:[5,4,1,""],PER_DATE_OF_DEATH:[5,4,1,""],PER_EMPLOYEE_OR_MEMBER_OF:[5,4,1,""],PER_MET_WITH:[5,4,1,""],PER_NATIONALITY:[5,4,1,""],PER_ORIGIN:[5,4,1,""],PER_OTHER_FAMILY:[5,4,1,""],PER_PARENTS:[5,4,1,""],PER_RELIGION:[5,4,1,""],PER_SCHOOLS_ATTENDED:[5,4,1,""],PER_SIBLINGS:[5,4,1,""],PER_SPOUSE:[5,4,1,""],PER_STATEORPROVINCE_OF_BIRTH:[5,4,1,""],PER_STATEORPROVINCE_OF_DEATH:[5,4,1,""],PER_STATESORPROVINCES_OF_RESIDENCE:[5,4,1,""],PER_VISITED:[5,4,1,""],PER_WEBSITE:[5,4,1,""],PER_WON_AWARD:[5,4,1,""],PHYS_Located:[5,4,1,""],PHYS_Near:[5,4,1,""],Personnel_Elect:[5,4,1,""],Personnel_EndPosition:[5,4,1,""],Personnel_Nominate:[5,4,1,""],Personnel_StartPosition:[5,4,1,""],Transaction_TransferMoney:[5,4,1,""],Transaction_TransferOwnership:[5,4,1,""]},"streamcorpus.Sentence":{labels:[5,4,1,""],tokens:[5,4,1,""]},"streamcorpus.StreamItem":{abs_url:[5,4,1,""],body:[5,4,1,""],doc_id:[5,4,1,""],external_ids:[5,4,1,""],original_url:[5,4,1,""],other_content:[5,4,1,""],ratings:[5,4,1,""],schost:[5,4,1,""],source:[5,4,1,""],source_metadata:[5,4,1,""],stream_id:[5,4,1,""],stream_time:[5,4,1,""],version:[5,4,1,""]},"streamcorpus.StreamTime":{epoch_ticks:[5,4,1,""],zulu_timestamp:[5,4,1,""]},"streamcorpus.Tagging":{generation_time:[5,4,1,""],raw_tagging:[5,4,1,""],tagger_config:[5,4,1,""],tagger_id:[5,4,1,""],tagger_version:[5,4,1,""]},"streamcorpus.Target":{kb_id:[5,4,1,""],kb_snapshot_time:[5,4,1,""],target_id:[5,4,1,""]},"streamcorpus.Token":{custom_entity_type:[5,4,1,""],dependency_path:[5,4,1,""],entity_type:[5,4,1,""],equiv_id:[5,4,1,""],labels:[5,4,1,""],lemma:[5,4,1,""],mention_id:[5,4,1,""],mention_type:[5,4,1,""],offsets:[5,4,1,""],parent_id:[5,4,1,""],pos:[5,4,1,""],sentence_pos:[5,4,1,""],token:[5,4,1,""],token_num:[5,4,1,""]},"streamcorpus_pipeline._clean_html":{clean_html:[6,2,1,""]},"streamcorpus_pipeline._clean_visible":{clean_visible:[6,2,1,""]},"streamcorpus_pipeline._convert_kba_json":{convert_kba_json:[6,2,1,""]},"streamcorpus_pipeline._dedup":{dedup:[6,2,1,""]},"streamcorpus_pipeline._docx_to_text":{docx_to_text:[6,2,1,""]},"streamcorpus_pipeline._dump_label_stats":{dump_label_stats:[6,2,1,""]},"streamcorpus_pipeline._filters":{debug_filter:[6,2,1,""],dump_stream_id_abs_url:[6,2,1,""],filter_domains:[6,2,1,""],filter_domains_substrings:[6,2,1,""],filter_languages:[6,2,1,""],filter_tagger_ids:[6,2,1,""],id_filter:[6,2,1,""],remove_raw:[6,2,1,""],replace_raw:[6,2,1,""]},"streamcorpus_pipeline._find":{find:[6,2,1,""],find_doc_ids:[6,2,1,""]},"streamcorpus_pipeline._fix_text":{fix_text:[6,2,1,""]},"streamcorpus_pipeline._guess_media_type":{file_type_stats:[6,6,1,""],guess_media_type:[6,2,1,""]},"streamcorpus_pipeline._handle_unconvertible_spinn3r":{handle_unconvertible_spinn3r:[6,2,1,""]},"streamcorpus_pipeline._hyperlink_labels":{hyperlink_labels:[6,2,1,""]},"streamcorpus_pipeline._john_smith":{john_smith:[6,2,1,""]},"streamcorpus_pipeline._kvlayer":{from_kvlayer:[6,2,1,""],to_kvlayer:[6,2,1,""]},"streamcorpus_pipeline._kvlayer.from_kvlayer":{"__call__":[6,3,1,""]},"streamcorpus_pipeline._language":{language:[6,2,1,""]},"streamcorpus_pipeline._lingpipe":{lingpipe:[6,2,1,""]},"streamcorpus_pipeline._local_storage":{from_local_chunks:[6,2,1,""],from_local_files:[6,2,1,""],to_local_chunks:[6,2,1,""],to_local_tarballs:[6,2,1,""]},"streamcorpus_pipeline._nilsimsa":{nilsimsa:[6,2,1,""]},"streamcorpus_pipeline._pdf_to_text":{pdf_to_text:[6,2,1,""]},"streamcorpus_pipeline._pipeline":{Pipeline:[6,2,1,""],PipelineFactory:[6,2,1,""]},"streamcorpus_pipeline._pipeline.Pipeline":{"__init__":[6,3,1,""],"_process_task":[6,3,1,""],run:[6,3,1,""]},"streamcorpus_pipeline._pipeline.PipelineFactory":{"__call__":[6,3,1,""],"__init__":[6,3,1,""],create:[6,3,1,""],lock:[6,4,1,""],registry:[6,4,1,""],tmp_dir_suffix:[6,4,1,""]},"streamcorpus_pipeline._rejester":{rejester_run_function:[6,6,1,""],rejester_terminate_function:[6,6,1,""]},"streamcorpus_pipeline._s3_storage":{from_s3_chunks:[6,2,1,""],to_s3_chunks:[6,2,1,""],to_s3_tarballs:[6,2,1,""]},"streamcorpus_pipeline._serif":{serif:[6,2,1,""]},"streamcorpus_pipeline._serifxml":{from_serifxml:[6,2,1,""]},"streamcorpus_pipeline._spinn3r_feed_storage":{from_spinn3r_feed:[6,2,1,""]},"streamcorpus_pipeline._taggers":{byte_offset_align_labels:[6,2,1,""],line_offset_align_labels:[6,2,1,""],name_align_labels:[6,2,1,""]},"streamcorpus_pipeline._title":{title:[6,2,1,""]},"streamcorpus_pipeline._tokenizer":{nltk_tokenizer:[6,2,1,""]},"streamcorpus_pipeline._upgrade_streamcorpus":{keep_annotated:[6,2,1,""],upgrade_streamcorpus:[6,2,1,""]},"streamcorpus_pipeline._upgrade_streamcorpus_v0_3_0":{upgrade_streamcorpus_v0_3_0:[6,2,1,""]},"streamcorpus_pipeline._yaml_files_list":{yaml_files_list:[6,2,1,""]},"streamcorpus_pipeline.force_clean_html":{force_clean_html:[6,2,1,""]},"streamcorpus_pipeline.offsets":{xpath_offsets:[6,2,1,""]},"streamcorpus_pipeline.stages":{BatchTransform:[6,2,1,""],Configured:[6,2,1,""],IncrementalTransform:[6,2,1,""],PipelineStages:[6,2,1,""],StageRegistry:[6,2,1,""]},"streamcorpus_pipeline.stages.BatchTransform":{process_path:[6,3,1,""],shutdown:[6,3,1,""]},"streamcorpus_pipeline.stages.Configured":{config:[6,4,1,""]},"streamcorpus_pipeline.stages.IncrementalTransform":{process_item:[6,3,1,""],shutdown:[6,3,1,""]},"streamcorpus_pipeline.stages.StageRegistry":{init_stage:[6,3,1,""],load_external_stages:[6,3,1,""],load_module_stages:[6,3,1,""],tryload_stage:[6,3,1,""]},"yakonfig.Configurable":{add_arguments:[7,3,1,""],check_config:[7,3,1,""],config_name:[7,4,1,""],default_config:[7,4,1,""],discover_config:[7,3,1,""],normalize_config:[7,3,1,""],replace_config:[7,3,1,""],runtime_keys:[7,4,1,""],services:[7,4,1,""],sub_modules:[7,4,1,""]},"yakonfig.ProxyConfigurable":{"__init__":[7,3,1,""]},"yakonfig.cmd":{ArgParseCmd:[7,2,1,""]},"yakonfig.cmd.ArgParseCmd":{add_arguments:[7,3,1,""],main:[7,3,1,""],runcmd:[7,3,1,""]},dblogger:{DatabaseLogHandler:[1,2,1,""],FixedWidthFormatter:[1,2,1,""],configure_logging:[1,6,1,""],query:[1,1,0,"-"]},kvlayer:{BadKey:[2,2,1,""],DatabaseEmpty:[2,2,1,""],client:[2,6,1,""]},rejester:{Registry:[4,2,1,""],RejesterQueue:[4,2,1,""],TaskMaster:[4,2,1,""],WorkUnit:[4,2,1,""],Worker:[4,2,1,""],exceptions:[4,1,0,"-"],run:[4,1,0,"-"],run_multi_worker:[4,1,0,"-"],workers:[4,1,0,"-"]},streamcorpus:{Annotator:[5,2,1,""],Attribute:[5,2,1,""],AttributeType:[5,2,1,""],Chunk:[5,2,1,""],ContentItem:[5,2,1,""],EntityType:[5,2,1,""],FlagType:[5,2,1,""],Gender:[5,2,1,""],Label:[5,2,1,""],Language:[5,2,1,""],MentionType:[5,2,1,""],Offset:[5,2,1,""],OffsetType:[5,2,1,""],Rating:[5,2,1,""],Relation:[5,2,1,""],RelationType:[5,2,1,""],Sentence:[5,2,1,""],StreamItem:[5,2,1,""],StreamTime:[5,2,1,""],Tagging:[5,2,1,""],Target:[5,2,1,""],Token:[5,2,1,""],VersionMismatchError:[5,2,1,""],add_annotation:[5,6,1,""],compress_and_encrypt:[5,6,1,""],compress_and_encrypt_path:[5,6,1,""],decrypt_and_uncompress:[5,6,1,""],deserialize:[5,6,1,""],get_date_hour:[5,6,1,""],get_entity_type:[5,6,1,""],make_stream_item:[5,6,1,""],make_stream_time:[5,6,1,""],serialize:[5,6,1,""]},streamcorpus_pipeline:{"_directory":[6,1,0,"-"],"_pipeline":[6,1,0,"-"],"_rejester":[6,1,0,"-"],run:[6,1,0,"-"],stages:[6,1,0,"-"]},yakonfig:{Configurable:[7,2,1,""],ConfigurationError:[7,2,1,""],NewSubModules:[7,2,1,""],ProgrammerError:[7,2,1,""],ProxyConfigurable:[7,2,1,""],check_subconfig:[7,6,1,""],check_toplevel_config:[7,6,1,""],clear_global_config:[7,6,1,""],defaulted_config:[7,6,1,""],diff_config:[7,6,1,""],get_global_config:[7,6,1,""],overlay_config:[7,6,1,""],parse_args:[7,6,1,""],set_default_config:[7,6,1,""],set_global_config:[7,6,1,""]}},objnames:{"0":["std","option","option"],"1":["py","module","Python module"],"2":["py","class","Python class"],"3":["py","method","Python method"],"4":["py","attribute","Python attribute"],"5":["py","classmethod","Python class method"],"6":["py","function","Python function"],"7":["py","exception","Python exception"]},objtypes:{"0":"std:option","1":"py:module","2":"py:class","3":"py:method","4":"py:attribute","5":"py:classmethod","6":"py:function","7":"py:exception"},terms:{"000123z":5,"01t12":5,"2c03a9507ae265ecf5b5356885a53393a2029d241394997265a1a25aefc6":2,"8112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb":2,"__call__":[4,6],"__init__":[4,6],"_abcol":6,"_abstract_storag":2,"_clean_html":6,"_clean_vis":6,"_cleanup":6,"_convert_kba_json":6,"_dedup":6,"_docx_to_text":6,"_dump_label_stat":6,"_except":2,"_filter":6,"_find":6,"_fix_text":6,"_guess_media_typ":6,"_handle_unconvertible_spinn3r":6,"_hyperlink_label":6,"_john_smith":6,"_kvlayer":6,"_languag":6,"_lingpip":6,"_local_storag":6,"_nilsimsa":6,"_path":4,"_pdf_to_text":6,"_pipelin":6,"_process_task":6,"_redi":4,"_rejest":6,"_s3_storag":6,"_serif":6,"_serifxml":6,"_spinn3r_feed_storag":6,"_tagger":6,"_task_mast":4,"_titl":6,"_token":6,"_upgrade_streamcorpu":6,"_upgrade_streamcorpus_v0_3_0":6,"_values_to_nam":5,"_yaml_files_list":6,"abstract":0,"boolean":[5,6],"break":2,"byte":[2,3,5,6],"case":[2,4,5,6],"char":[3,4,5],"class":[2,4,5,6],"default":[2,3,4,5,6],"enum":5,"export":2,"final":6,"float":[2,4,5],"function":[2,4,6],"import":[4,5,6],"int":[2,4,5,6],"long":[2,4,5],"new":[4,5,6],"null":6,"public":5,"return":[2,4,5,6],"short":[4,5],"super":6,"switch":4,"true":[2,3,4,5,6],"try":[4,6],"while":[2,4,6],a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0:2,a_stag:6,abl:[2,4],about:[4,5,6],abov:[2,4,6],abs_url:[5,6],absent:4,absolut:[4,6],absolute_filenam:6,absolute_path:6,abstractstorag:2,accept:6,accept_stream_id:6,access:[2,4,5,6],access_key_id:4,accord:3,accumul:2,accumulo:0,accumulo_latency_m:2,accumulo_max_memori:2,accumulo_thread:2,accumulo_timeout_m:2,acquir:4,acress:6,across:[2,4,5,6],act:6,activ:2,actual:[2,4,5,6],add:[2,4,5,6],add_annot:5,add_dependent_task:4,add_dependent_work_unit:4,add_item:4,add_work_unit:4,addit:[2,4,5,6],addition:[4,6],address:[4,5],affect:[2,4],after:[2,4,6],afterward:[4,6],again:[4,5,6],against:[4,6],aid:5,align:6,align_labels_bi:[3,6],aligner_data:[3,6],aliv:[2,4],all:[0,2,3,4,5,6],all_domain:[3,6],allow:[2,4,5,6],allow_mult:2,allow_null_languag:6,almost:6,along:4,alphanumer:2,alreadi:[2,4,6],also:[2,4,5,6],altern:[4,6],alternatlei:6,alwai:[2,4,5,6],amazon:[2,5,6],amazonaw:2,ambigu:5,amount:4,anchor:5,ani:[0,2,3,4,5,6],annot:[3,5,6],annotation_tim:5,annotator_id:[3,5,6],anoth:4,any_multi_token:3,anybodi:4,anyth:[2,4],anywher:4,apach:[2,3,5],api:0,app:[2,4],app_nam:[2,3,4],appear:[3,5],append:[2,6],appli:[2,5,6],applic:[2,4,6],apply_async:4,appname_namespace_t:2,appropri:[4,5,6],approxim:5,arbitrari:5,arbitrarili:4,archiv:6,aren:[4,6],arg:[2,4,5,6],argpars:6,argument:[2,4,6],argumentpars:6,around:[2,6],arrai:5,art_userownerinventormanufactur:5,artefact:5,articl:[5,6],as_child:4,ask:[0,3,4],assembl:5,assert:[5,6],assert_single_sourc:6,assessor:5,assign:[4,5],assist:5,associ:[4,5,6],assum:[2,4,6],assumpt:6,astag:6,atim:4,atom:[2,4],attach:[4,5,6],attempt:[4,6],attribut:5,attribute_typ:5,attributetyp:5,author:[5,6],auto:5,autofactori:6,automat:[4,5,6],automethod:4,auxiliari:6,avail:[2,4,5,6],available_gb:4,averag:4,avoid:[4,5],awai:6,awaken:4,aws_access_key_id:[2,4,6],aws_access_key_id_path:[2,4,6],aws_secret_access_kei:[2,4,6],aws_secret_access_key_path:[2,4,6],axxxxxxxxxxxxxxxxxxx:2,b_stage:6,back:[2,3,4,6],backend:0,background:4,badkei:2,base:[0,2],basenam:6,basic:[2,3,4,5,6],batch:4,batch_transform:[3,4,6],batchtransform:6,becaus:[4,6],becom:4,been:[4,6],befor:[4,6],begin:2,behalf:4,behav:4,behavior:[2,6],below:[2,4,5],benefici:4,best:[4,5],better:[2,4,6],between:[2,4,5,6],beyond:4,big:6,bin:[2,6],binari:[2,5],blank:5,blob:5,block:[2,4,6],blunt:5,bob:5,bodi:[4,5,6],bool:[4,6],borrow:5,both:[2,4,5],boto:2,bound:4,bridg:4,broadli:6,broken:[2,5,6],browser:6,bucket:[2,4,6],buffer:[2,5],built:4,bulk:2,bulki:6,bunch:4,bundl:6,busi:6,business_declarebankruptci:5,business_endorg:5,business_mergeorg:5,business_startorg:5,by_tim:6,byte_offset_align_label:6,call:[2,4,6],callabl:6,caller:[4,6],came:4,can:[0,2,3,4,5,6],can_start_mor:4,cannot:[4,6],canon:5,canonic:5,canonical_nam:[3,6],cat:4,categori:6,caus:[3,4,6],cbor:4,cborchunk:5,cell:2,central:4,certain:4,chain:5,chain_selector:[3,6],chang:[2,4,5,6],charact:[2,5,6],charg:6,check:[2,4,6],check_out_item:4,check_put_key_valu:2,check_spinning_children:4,checksum:6,chicago:5,child:[4,6],child_lifetim:4,children:4,choic:6,chosen:5,chrome:5,chunk:[2,3,5,6],chunk_path:6,circumst:6,civil:5,classif:5,classmethod:4,clean:[4,6],clean_html:[3,5,6],clean_vis:[3,4,5,6],cleans:6,cleanup:[4,6],cleanup_tmp_fil:[3,4,6],clear:4,clear_signal_handl:4,clear_tabl:2,client:[2,4],close:[2,5,6],cluster:[2,4,6],code:[3,4,5,6],collect:[2,4,5,6],com:[0,2,3,4,6],combin:[2,4,6],come:[3,4,5,6],command:[4,6],comment:5,common:[2,3,5,6],compare_md5_in_file_nam:6,compat:4,compil:4,complet:[4,5,6],complic:6,compon:0,composit:6,compress:[5,6],compress_and_encrypt:5,compress_and_encrypt_path:5,comput:[3,6],conceiv:4,concurr:[4,6],config:[2,3,4,6],config_nam:[4,6],configur:[0,2,3],configurationerror:2,confirm:4,conflict_attack:5,conflict_demonstr:5,confus:4,conigur:6,connect:[2,4,6],connection_pool_s:2,consid:[4,6],consist:[2,4,5,6],constant:4,construct:[2,5,6],constructor:[2,4,6],consum:4,contact:4,contact_meet:5,contact_phonewrit:5,contain:[2,4,5,6],contains_ment:5,content:[0,4,5,6],content_extract:6,content_form:5,content_item:5,contentitem:5,context:[4,6],continu:4,control:[4,6],conveni:[3,6],convent:5,convention:4,convert:[3,4,5,6],convert_kba_json:6,copi:[2,4,5,6],copy_to_filenam:2,coref:5,corenlp:5,corpora:5,corpu:[5,6],correct:[2,4,5],correl:5,correspond:[2,3,4,5,6],correspondingli:[2,4,6],cost:2,could:[4,6],count:4,counter:2,coupl:4,cours:5,cpu:4,crash:4,creat:[2,3,4,5,6],credenti:[2,6],criteria:6,current:[2,4,5,6],custom:[3,6],custom_entity_typ:5,custom_typ:5,cycl:4,daemon:4,data:[0,2,3,4],data_item:5,databas:0,databaseempti:2,dataset:3,date:[5,6],date_hour:[4,5,6],davi:5,david:5,dblogger:0,dbname:2,dead:4,deadlin:4,debug:[4,5,6],debug_filt:6,debug_work:4,declar:6,decod:[4,6],decompress:6,decrypt:[4,5,6],decrypt_and_uncompress:5,dedic:[4,6],dedup:6,deeper:2,def:[4,6],default_config:6,default_lifetim:4,defin:[4,5,6],definit:[2,4,6],definiton:4,del:4,del_work_unit:4,delet:[2,4,6],delete_namespac:2,deliv:4,delta:2,demand:5,depend:0,dependency_path:5,depends_on:4,deprec:4,depriorit:6,deriv:[5,6],desc:[4,6],describ:[2,4,5,6],descript:[4,5,6],deseri:5,desir:6,destin:4,detail:2,detect:[4,5,6],detect_compress:5,determin:[2,5],develop:[5,6],diagnost:4,dict:[2,4,6],dict_nam:4,dictionari:[2,4,5,6],did:[4,6],didn:4,die:4,diffeo:[0,3,6],differ:[2,4,5,6],direct:4,direct_cal:4,directli:[2,4,5,6],directori:[2,3,4,5,6],disabl:6,discov:5,disk:[5,6],dispatch:4,distinct:[4,6],distinguish:5,distribut:[0,2,3],do_run_on:4,do_some_work:4,do_stag:6,doc:[5,6],doc_id:[5,6],doc_ids_8:6,doc_path:[3,6],docker:4,document:[0,3,4],docx_to_text:6,doe:[0,2,3,4,6],doesn:[2,4],domain:[3,6],domain_substr:6,don:[4,6],done:[3,4],dossier:6,dot:6,down:[4,6],download:[5,6],drop:[2,6],dump:[3,4,6],dump_label_stat:6,dump_queu:4,dump_stream_id_abs_url:6,dynam:3,e8160039594a33894f6564e1b1348bbd7a0088d42c4acb73eeaed59c009d:2,each:[2,3,4,5,6],earlier:4,earliest:5,easi:4,easili:[2,4,6],ebf059defa41b2812c1f969f28cdb45:6,ec2:4,edu:5,effect:[2,4],effici:[4,6],effort:[4,6],either:[2,3,4,5,6],eleph:5,els:[2,4],email:5,embed:[2,3],emit:6,empti:[2,3,4,5,6],enabl:[2,4,6],encod:[4,5,6],encourag:2,encrypt:[4,5,6],end:[2,4,5,6],end_tim:2,endian:6,enforc:4,engin:4,english:[5,6],enough:[2,4],enough_memori:4,ensur:[4,6],entir:[4,5,6],entiti:[3,5,6],entity_typ:5,entitytyp:5,entri:6,enumer:5,environ:[3,4,6],environmenterror:4,epoch:6,epoch_tick:[5,6],equal:[2,4],equiv_id:5,equival:[2,5],error:[2,4,6],escap:5,essenti:6,even:[2,4,5,6],event:5,ever:4,everi:[2,4,6],everyth:4,evid:5,exact:4,exactli:[4,5,6],examin:4,exampl:[0,2],exc:4,exceed:4,except:0,exclud:6,excluded_doc_id:6,excluded_doc_ids_path:6,excluded_stream_id:6,excluded_stream_ids_path:6,execut:[0,4],exist:[2,4,5,6],exit:[4,6],exit_bor:4,exit_except:4,exit_success:4,expand:[2,5,6],expect:[2,4,5,6],expens:[4,6],experi:2,experienc:4,expir:4,expiri:4,explicitli:[4,5],explor:4,extend:4,extens:[0,3,6],extern:[4,5,6],external_id:5,external_profil:6,external_stages_modul:6,external_stages_path:[4,6],extract:[5,6],extractor:[0,3],extrem:6,fac:5,fac_:5,fac_loc:5,fac_own:5,fac_visited_bi:5,fact:4,factori:[0,3,6],fail:[2,4,6],failur:[4,6],fairli:4,fall:6,fall_2003:5,fallback_media_typ:6,fals:[2,4,5,6],fashion:4,faster:2,fastest:6,featur:6,featurecollect:6,fed:6,feed:6,femal:5,fetch:[2,4,6],few:6,fewer:2,field:[2,4,5,6],figur:4,file2:4,file:[0,2,3,4,5,6],file_type_stat:6,filenam:[2,6],filesystem:6,fill:[4,5,6],filter:0,filter_domain:6,filter_domains_match:6,filter_domains_substr:6,filter_exact_match:4,filter_languag:6,filter_tagger_id:6,find:[2,3,4,6],find_doc_id:6,finish:[4,6],first:[2,4,5,6],five:[2,4,6],fix:[2,4,6],fix_text:6,flag:5,flagtyp:5,flat:[0,3,6],flow:[3,4,6],flush:5,focus:5,follow:[2,3,4,5,6],forc:6,force_clean_html:6,force_clear_lock:4,forev:4,fork:4,fork_work:[4,6],forkwork:4,form:[2,4,5,6],format:[2,3,4,5,6],found:[2,4],four:[2,6],fqdn:4,fraction:5,fragment:6,framework:0,free:5,freebas:5,frequent:5,friend:5,from:[2,3,4,5,6],from_dict:4,from_kvlay:6,from_local_chunk:6,from_local_fil:6,from_s3_chunk:[4,6],from_serifxml:6,from_spinn3r_fe:6,full:[5,6],fulli:6,functionnam:6,further:[4,5,6],furthermor:4,futur:[4,6],garbag:5,gconfig:4,genaff_citizenresidentreligionethn:5,genaff_orgloc:5,gender:5,gener:[2,3,4,5,6],generation_tim:5,get:[2,4,6],get_assigned_work_unit:4,get_child_work_unit:4,get_date_hour:5,get_entity_typ:5,get_global_config:[4,6],get_heartbeat:4,get_mod:4,get_work:4,get_work_spec:4,get_work_unit:4,get_work_unit_statu:4,getitem_reset:4,gigabyt:4,github:[0,3,4],give:[2,5,6],given:[2,4,5,6],glob:6,global:[0,2,4],global_config:4,global_lock:4,gnupg:5,good:4,googl:[0,3],googlegroup:4,got:4,gpe:5,gpg:[4,6],gpg_decryption_key_path:[4,6],gpg_encryption_key_path:[4,6],gpg_key_path:5,gpg_privat:5,gpg_public:5,gpg_recipi:[4,5,6],grammat:5,greater:4,greatest:6,group:[0,3,4],guarante:[2,6],guess:[5,6],guess_media_typ:[3,6],guidelines_v5:5,gzip:6,had:[2,4],handle_unconvertible_spinn3r:6,handler:4,happen:4,hard:4,hash:[2,5,6],hash_frequ:6,hash_keyword:6,hash_tf_sid:6,have:[2,4,5,6],haven:4,header:5,headlesswork:4,heartbeat:4,heartbeat_interv:4,held:4,help:6,helper:5,here:[2,3,4,5,6],hex:6,hexdigest:6,hidden:4,hierarchi:[2,6],high:4,higher:[2,4],highest:4,histor:2,hold:[2,4,5,6],holder:4,home:6,honor:4,hopefulli:4,host:[0,2,3,4],hostnam:[4,5,6],hour:6,how:[2,3,4,6],howev:[4,6],href:6,html:[5,6],http:[2,3,5,6],http_header:5,http_port:2,human:5,hundr:4,hybrid:2,hyperlink:[5,6],hyperlink_label:[3,6],hyphen:6,i_str:6,id_filt:6,ident:[4,5],identif:4,identifi:[4,5,6],idl:4,idle_all_work:4,if_clean_visible_remove_raw:6,ignor:[2,4,6],illustr:4,immedi:[2,4],implement:[2,3],implementd:4,impli:[5,6],importerror:6,inbound:6,includ:[2,3,4,5,6],include_domain:6,include_domains_path:6,include_language_cod:6,include_mime_typ:6,include_prior:4,include_substr:6,include_substrings_path:6,included_doc_id:6,included_doc_ids_path:6,included_language_cod:6,included_stream_id:6,included_stream_ids_path:6,incompat:2,inconsist:4,increment:[2,4],incremental_transform:[3,4,6],incrementaltransform:6,independ:6,index:[0,2,5,6],individu:[4,5,6],inf:[2,4],info:[3,4,5,6],inform:[4,5,6],infrastructur:4,init_stag:6,initi:[2,4,6],inject:6,inplac:6,input:[3,4,5,6],input_fnam:6,input_format:[4,6],input_item_limit:6,input_md5:[4,6],insert:[4,5],insid:2,inspect:[4,6],inspect_work_unit:4,instal:[2,6],instanc:[2,4,5,6],instanti:[2,6],instead:[4,5,6],instruct:4,integ:[2,4,5,6],integr:[0,4],intend:[2,4],interact:4,interest:4,interfac:[3,4,5],intermedi:6,intern:[2,5],interpret:[4,5,6],interv:4,invalid:[2,4],invas:4,invoc:0,invok:4,involv:4,is_priv:6,isn:4,item:[0,2,3,4],iteminuseerror:4,iter:5,iter_work_spec:4,itself:[0,2,3,4,6],jan:5,job:[4,6],john:6,john_smith:6,join:4,json:[4,5],jsonchunk:5,judg:5,just:[2,4,5,6],justice_acquit:5,justice_app:5,justice_arrestjail:5,justice_chargeindict:5,justice_convict:5,justice_execut:5,justice_extradit:5,justice_fin:5,justice_pardon:5,justice_releaseparol:5,justice_sent:5,justice_su:5,justice_trialhear:5,kb_id:5,kb_snapshot_tim:5,kba:[0,3,4,5,6],kbp2013:5,kbp:5,keep:[2,4],keep_annot:6,kei:0,key1:4,key2:4,key_rang:2,key_spec:2,keyerror:6,keys_and_valu:2,keys_siz:2,keyword:6,keyword_tagger_id:6,kick:4,kill:[4,6],know:[2,6],knowledg:[5,6],knowledge_bas:6,known:[3,4,5,6],kv_appname_namespac:2,kvlayer:0,kvlayer_prefix:2,kwarg:[2,4,5,6],kwd:4,label:[3,5,6],lack:6,languag:[0,3,5,6],larg:[2,4,5],last:[4,6],latenc:2,later:[4,6],latest:[3,4,6],launch:4,ldc:5,leak:4,leas:4,lease_tim:4,least:[4,6],left:[4,6],legal:6,lemma:5,lemmat:5,len:4,length:[2,5],less:[2,4,6],lest:4,letter:[2,5],level:[2,3,4,5,6],leveldb:2,librari:2,life_beborn:5,life_di:5,life_divorc:5,life_injur:5,life_marri:5,lifetim:4,lightweight:4,like:[2,3,4,5,6],limit:[2,4],line2:6,line:[4,5,6],line_offset_align_label:6,ling001:5,ling:5,lingpip:6,link:4,list:[2,4,5,6],list_available_work_unit:4,list_blocked_work_unit:4,list_failed_work_unit:4,list_finished_work_unit:4,list_of_s3_path:4,list_pending_work_unit:4,list_work_spec:4,list_work_unit:4,liter:6,littl:4,live:4,live_log_child:4,load:[3,4,6],load_external_stag:6,load_module_stag:6,loc:5,local_storag:2,localhost:2,locat:[2,4,5],lock:[4,6],lockerror:4,log:0,log_delet:2,log_get:2,log_put:2,log_scan:2,log_scan_kei:2,log_spew:4,logger:6,logpath:4,longer:4,look:[2,3,4,6],loop:4,loos:5,lost:4,lostleas:4,lot:4,low:[2,4],lower:4,lowest:4,ltime:4,lzma:6,machin:4,made:4,mai:[2,4,5,6],main:4,maintain:[4,6],make:[4,5,6],make_stream_item:5,make_stream_tim:5,male:5,manag:0,mandatori:6,mani:[2,4,5,6],manipul:4,manual:[4,6],map:[2,4,5,6],mark:[3,4,5],massiv:[0,3],master:4,match:[2,3,4,5,6],match_str:4,match_strings_path:4,matcher:4,max:4,max_backoff:6,max_connect:2,max_consistency_delai:2,max_job:4,max_retri:6,maximum:4,mayb:4,md5:[4,5,6],mean:[3,4,5,6],meaning:4,mechan:4,media:6,media_typ:[5,6],member:6,memori:[2,4,5],mention:[3,5,6],mention_id:[5,6],mention_id_1:5,mention_id_2:5,mention_typ:5,mentiontyp:5,mere:4,messag:[4,5,6],metadata:[5,6],method:[2,4,6],mid:4,midnight:5,might:[2,5,6],migrat:6,mildli:4,million:4,mime:[5,6],min_connect:2,min_gb:[4,6],minim:[5,6],minimum:4,minut:[2,4],mirror:6,misc:5,miss:4,mod:6,mode:[4,6],mode_count:4,model:2,modif:6,modifi:[4,6],modul:[0,2],modulenam:6,monei:5,monitor:4,more:[0,2,3,4,6],most:[4,5,6],move:[4,6],move_al:4,movement_transport:5,msg:5,much:[2,6],multi:[4,5],multi_work:4,multipl:[2,4,5],multiprocess:4,multiwork:4,must:[2,4,5,6],mutablemap:6,my_kba_filt:4,my_match_str:4,mybucket:2,mydataset:3,mynamespac:4,name:[2,4,5,6],name_align_label:6,names_in_chain:[3,6],namespac:[2,3,4,6],nation:5,natur:[0,3,4,5],necessari:[3,6],need:[2,3,4,5,6],neg:[4,5],neither:4,ner:[4,5],network:[2,4,5,6],neutral:5,never:[2,4,6],new_prior:4,newer:[2,4],newlin:4,next:4,nice:[4,6],nilsimsa:6,nist:[4,5],nlp:[3,5,6],nltk:6,nltk_token:6,nobodi:4,node:2,nom:5,nomin:6,non:[2,4,6],none:[2,4,5,6],nonexist:4,nor:4,normal:[2,4,5,6],nosuchworkspecerror:4,nosuchworkuniterror:4,note:[2,4,5,6],noth:[2,4],now:[4,6],num:[4,6],num_avail:4,num_block:4,num_fail:4,num_finish:4,num_kei:2,num_pend:4,num_task:4,num_valu:2,num_work:4,num_workers_per_cor:4,number:[2,4,5,6],numer:[2,4,5],object:[2,4,5,6],observ:[4,5],obtain:[4,6],obviou:[4,5],occur:4,off:[2,4],offset:[5,6],offset_typ:[3,6],offsettyp:5,often:[4,5],omit:6,onc:[4,6],onli:[2,4,5,6],opaqu:5,open:[4,6],oper:[0,2,3,4,6],opt:6,option:[2,4,5,6],ord:4,order:[2,4,5],orderli:[4,6],org:[5,6],org_:5,org_alternate_nam:5,org_attend:5,org_city_of_headquart:5,org_country_of_headquart:5,org_date_dissolv:5,org_date_found:5,org_founded_bi:5,org_memb:5,org_member_of:5,org_number_of_employees_memb:5,org_par:5,org_political_or_religious_affili:5,org_sharehold:5,org_stateorprovince_of_headquart:5,org_subsidiari:5,org_top_members_employe:5,org_visit:5,org_websit:5,orgaff_employ:5,orgaff_found:5,orgaff_investorsharehold:5,orgaff_membership:5,orgaff_ownership:5,orgaff_sportsaffili:5,orgaff_studentalum:5,organ:5,orient:2,origin:[3,4,5,6],original_url:5,other:[2,4,5,6],other_cont:[5,6],otherdir:6,otherwis:[4,6],our:4,ourselv:4,out:[4,6],outgo:6,output:[0,3,4,5,6],output_chunk_max_count:[3,6],output_format:6,output_max_clean_visible_byt:6,output_nam:[4,6],output_path:6,output_typ:6,outstand:4,over:[2,3,5,6],overdu:4,overlaid:2,overlap:5,overrid:[2,4],overview:0,overwrit:4,owlp:5,own:[2,3,4,5],pack:2,packag:[0,3,4,6],page:0,pair:[2,4],par:[3,6],par_addit:6,parallel:4,param:[5,6],paramet:[2,4,5,6],parent:[4,5,6],parent_id:5,parent_item:4,pars:[5,6],parse_arg:6,parser:6,part:[2,4,5,6],parti:[0,3,6],partial:[4,5,6],particular:[2,4,5],particularli:[2,6],partit:2,partwhole_artifact:5,partwhole_geograph:5,partwhole_subsidiari:5,pass:[2,3,4,5,6],password:[2,3],past:3,path:[2,3,4,5,6],path_in_third:[3,6],path_prefix:2,pattern:6,pb_port:2,pbc:2,pdf:5,pdf_to_text:6,pend:4,penn_treebank_po:5,per:[2,4,5,6],per_:5,per_ag:5,per_alternate_nam:5,per_attend:5,per_cause_of_death:5,per_charg:5,per_children:5,per_cities_of_resid:5,per_city_of_birth:5,per_city_of_death:5,per_countries_of_resid:5,per_country_of_birth:5,per_country_of_death:5,per_date_of_birth:5,per_date_of_death:5,per_employee_or_member_of:5,per_gend:5,per_met_with:5,per_nation:5,per_origin:5,per_other_famili:5,per_par:5,per_religion:5,per_schools_attend:5,per_sibl:5,per_spous:5,per_stateorprovince_of_birth:5,per_stateorprovince_of_death:5,per_statesorprovinces_of_resid:5,per_titl:5,per_visit:5,per_websit:5,per_won_award:5,percent:5,perform:[2,4],period:4,permit:[2,4],persist:[2,4],persoc_busi:5,persoc_famili:5,persoc_lastingperson:5,person:[5,6],personnel_elect:5,personnel_endposit:5,personnel_nomin:5,personnel_startposit:5,phone:5,phys_loc:5,phys_near:5,physic:5,pick:6,picklechunk:5,pid:4,pidfil:4,pipelin:[0,4,5],pipelinefactori:6,pipelinestag:6,piplin:[0,3],place:[2,4,5,6],plain:[3,6],plu:[4,5,6],poici:6,point:[2,4,5,6],pointer:2,polici:6,poll:4,poll_interv:4,pool:2,poorli:6,pop:4,popitem:4,popitem_mov:4,popmani:4,popul:[4,5,6],popular:2,port:[2,4],portion:5,posit:[5,6],possibl:[2,4,5],post:6,post_batch_incremental_transform:6,postgresql:2,potenti:4,practic:4,preced:[2,4,6],predecessor:4,prefetch:6,prefix:6,preload:6,prepend:6,presenc:4,present:[2,4,5,6],prevent:4,previou:[4,5],previous:6,primari:5,primarili:5,princip:4,print:[4,5,6],priorit:4,prioriti:0,priority_max:4,priority_min:4,priorityrangeempti:4,privat:[4,6],pro:5,probabl:4,problem:[4,6],process:[0,2,3,4,5],process_item:6,process_path:6,processor:4,produc:[3,4,5,6],profil:[5,6],profile_id:6,program:[3,4,6],programmat:6,programmererror:4,progress:[4,5,6],project:5,prompt:4,properti:[2,4,6],protect:6,protocol:[2,5],prototyp:4,proven:2,provid:[0,2,3,4,5,6],proxi:[2,3],psycopg2:2,pub:[4,6],publicdataset:[4,6],publish:6,publisher_typ:6,pull:4,punctuat:4,pure:[4,5],purg:4,purpos:[2,5],put:[2,4,6],python:0,qualifi:[4,6],quei:2,queri:[2,4,5],question:4,queue:0,quickli:4,quiet:[4,6],quot:4,rais:[2,4,6],ram:4,ran:4,rand8:6,random:[4,6],rang:[2,5,6],rank:5,rare:6,rate:[5,6],rate_log_interv:6,rather:[4,5,6],raw:[4,5,6],raw_tag:[5,6],re_acquire_lock:4,reach:4,react:4,read:[2,3,4,6],read_from:6,read_lock:4,read_msg_impl:5,read_serifxml:6,readabl:5,reader:[3,4],real:4,realli:4,reason:4,receiv:[2,4],recent:4,recipi:6,recogn:5,recommend:[5,6],reconnect:2,record:[4,5,6],recreat:4,recurs:6,redi:0,redis_db_num:2,redis_port_6379_tcp_addr:4,redis_port_6379_tcp_port:4,redisbas:4,redund:5,refer:[5,6],reflect:4,refresh:4,refus:4,regardless:6,regex:4,regist:4,registri:[4,6],registry_address:[3,4],reject:6,rejest:[0,3],rejester_run:4,rejester_run_funct:6,rejester_sampl:4,rejester_termin:4,rejester_terminate_funct:6,rejester_work:0,rejesterexcept:4,rejesterqueu:4,rel:[4,6],relai:2,relat:5,relation_typ:5,relationship:5,relationtyp:5,releas:4,relev:[4,5],reliabl:[4,6],religion:5,remain:[4,5],remot:4,remov:[4,6],remove_available_work_unit:4,remove_blocked_work_unit:4,remove_failed_work_unit:4,remove_finished_work_unit:4,remove_pending_work_unit:4,remove_raw:6,renam:6,renew_item:4,repeat:4,replac:[4,5,6],replace_raw:6,replication_factor:2,report:4,repres:5,represent:0,repriorit:4,request:4,requir:[2,4,6],require_abs_url:[3,6],require_clean_html:6,require_language_cod:6,reserv:4,reservations_item:4,reserve_item:4,reset_al:4,reset_prior:4,resid:6,resolv:5,resort:4,respect:2,respons:4,rest:6,restart:[4,6],result:[2,4,6],retri:[2,4,6],retriev:[4,6],retry_interv:2,return_item:4,reus:6,riak01:2,riak02:2,riak03:2,riak04:2,riak05:2,root:[2,3,4,6],root_path:[3,4,6],round:2,row:2,rsa:4,run:[0,2,3,4,6],run_funct:[4,6],run_on:4,run_work:4,runtim:4,s3_path_prefix:[4,6],safe:4,sai:4,saltstack:4,same:[2,4,5,6],samedir:6,sampl:[4,6],sane:6,satisfi:4,save:[2,4,5],scale:2,scan:[2,6],scan_inner_limit:2,scan_kei:2,scan_limit:2,schema:[2,5],scheme:[5,6],schost:[5,6],score:[4,5,6],scp_config:6,scram:4,search:[0,2,3],second:[2,4,5,6],secondari:2,secret:[2,3],secret_access_kei:4,section:[2,4,6],see:[2,3,4,5,6],seem:[2,6],select:[2,4,6],self:[4,6],semant:[4,6],senat:5,send:[4,6],sens:6,sent:6,sentenc:[5,6],sentence_blob:5,sentence_id:5,sentence_id_1:5,sentence_id_2:5,sentence_po:5,separ:[2,4,5,6],sequenc:[4,6],sequenti:6,seri:[4,6],serial:[2,4,5],serialize_si:6,serialz:2,serif:[0,3,6],serif_ex:6,serifxml:6,server:[2,4],session:4,set:[2,4,5,6],set_1to1:4,set_mod:4,set_signal_handl:4,set_titl:4,set_work_spec:4,setup:0,setup_namespac:2,sever:[4,5,6],sha:2,share:[2,3,4,6],shell:[4,6],shelv:2,shortcut:4,shot:6,should:[4,5,6],show:5,shown:[2,4,6],shut:4,shutdown:[4,6],sigabrt:4,sigint:4,signal:4,sigpip:4,sigterm:4,silent:4,similar:[4,5],similarli:4,simpl:[0,2],simpler:5,simpli:2,sinc:[4,5,6],singl:[0,2,4],singlework:4,skip:4,slice:4,slightli:2,slot:[3,4,5,6],slowest:6,small:4,smaller:4,smith:6,snappi:6,snapshot:5,social:5,softwar:0,sole:5,some:[2,3,4,5,6],somebodi:4,someon:4,someth:[2,4],sometim:5,somewher:6,soon:4,sort:6,sourc:[2,3,4,5,6],source_metadata:5,space:[4,6],spawn:[4,6],spawn_interv:4,spec:[4,6],special:[4,5,6],specif:[2,3,4,5,6],specifi:[2,4,6],specifif:2,speech:[5,6],spin:4,spinn3r:6,split:2,splitlin:4,sql:2,stabl:4,stage:[0,3],stage_cl:6,stage_config:6,stagenam:6,stageregistri:6,stai:4,stale:4,stamp:[4,6],standalon:6,standard:[2,4,6],standoff:6,stanford:5,start:[2,4,5],start_chunk_tim:6,start_count:6,start_log_child:4,start_tim:2,startup:4,stat:6,state:4,statement:2,statu:4,status:4,stdout:6,step:[4,6],still:[4,6],stop:[4,6],stop_all_children:4,stop_gracefulli:4,stop_jobs_earli:4,stop_log_child:4,storag:[0,2,3,4],storage_address:[2,3],storage_typ:[2,3],store:0,str:[2,4,5,6],str_to_valu:2,strategi:4,stream:[0,3],stream_id:[5,6],stream_item:[2,6],stream_items_t:6,stream_th:5,stream_tim:5,streamcorpus_directori:[0,4],streamcorpus_dump:0,streamcorpus_generate_serifxml:6,streamcorpus_one_step:[3,6],streamcorpus_pipelin:[0,3,4],streamcorpus_read_serifxml:6,streamcorpus_vers:[4,6],streamitem:[3,4,5,6],streamtim:5,string:[2,4,5,6],strip:[4,6],strip_punctu:4,strip_str:4,structur:[2,5,6],stuff:2,sub:4,subcommand:4,subdirectori:6,submit:[4,6],subordin:6,subprocess:4,subsequ:6,subset:4,substitut:6,substr:6,succe:4,success:6,successfulli:4,successor:4,suffici:4,suffix:6,suggest:4,suitabl:4,summar:3,summari:4,suppli:[2,6],support:[2,3,4,5,6],suppress_failur:6,surdeanu:5,sure:[4,5],syntax:4,system:[0,2],table_nam:2,tabular:4,tac:5,tac_2013_kbp_slot_descriptions_1:5,tag:[4,5,6],tagger:[3,4,5,6],tagger_config:5,tagger_id:[5,6],tagger_ids_to_keep:6,tagger_vers:5,take:[2,4,6],taken:6,talk:[2,4],tarbal:6,target:[4,5,6],target_id:[3,5,6],target_nam:6,tarinfo_gnam:6,tarinfo_nam:6,tarinfo_unam:6,task:0,task_mast:4,taskmast:[4,6],tasks_per_cpu:4,tell:[3,4],tempor:3,temporari:6,term:[2,3],termin:[4,6],terminate_funct:[4,6],test:[2,6],test_output:6,text:[0,3,4,5,6],textual:5,than:[2,4,5,6],thei:[4,5,6],them:[3,4,6],themselv:4,therefor:5,thi:[2,3,4,5,6],thing:4,think:4,third:[0,3,4,6],third_dir_path:[3,6],those:[2,4,6],though:[2,6],thought:6,thousand:4,thread:6,three:[4,6],thrift:[5,6],thrift_framed_transport_size_in_mb:2,through:[2,4,5,6],time:[2,4,5,6],timeout:4,timestamp:6,titl:[4,5,6],tmp:[2,3,4,5,6],tmp_dir:5,tmp_dir_path:[3,4,6],tmp_dir_suffix:6,to_dict:4,to_kvlay:[3,6],to_local_chunk:6,to_local_tarbal:6,to_s3_chunk:[4,6],to_s3_tarbal:6,togeth:6,tok:5,token:[5,6],token_num:5,too:4,tool:0,toolkit:[0,3],top:[2,4,6],topic:5,total:4,traceabl:4,traceback:4,tradat:2,tradit:2,transaction_transfermonei:5,transaction_transferownership:5,transform:5,translat:[4,6],trec:[0,3,4,5,6],tri:[4,6],trip:2,trivial:[2,4],troublesom:6,tryload_stag:6,ttype:5,tupl:[2,4,5],turn:6,two:[2,4,5,6],txt:[4,6],type:[2,4,5,6],typic:[2,4,5,6],ultim:6,unavail:5,uncategor:5,unchang:6,uncompress:5,undecod:6,under:[2,3,4,6],underli:[2,4],underneath:6,underscor:2,undo:4,unexpect:5,unformat:4,unicod:[4,5,6],unifi:[0,3],unind:5,union:6,uniqu:[4,5],unit:[4,5,6],unitnam:4,unix:[5,6],unknown:6,unless:[4,6],unlock:4,unmodifi:6,unprocess:5,unregist:4,unset:[4,6],unspecifi:2,unsuccessfulli:4,until:4,unus:3,unusu:6,updat:[2,4],update_bundl:[4,6],upenn:5,upgrade_streamcorpu:6,upgrade_streamcorpus_v0_3_0:6,upload:6,uri:6,url:[5,6],url_prefix:6,usag:[4,5],use_prefetch:6,user:[2,4,6],usernam:[2,3],usual:[4,5,6],utc:5,utf8:6,utf:[4,5,6],util:5,uuid:[2,4,6],v0_1_0:6,v0_2_0:6,v0_3_0:[4,5,6],val:2,valid:[4,5,6],validate_work_spec:4,valu:0,value_to_str:2,value_typ:2,values_s:2,variabl:[4,6],variou:[4,5],veh:5,verbos:[4,6],veri:[4,6],verif:6,verifi:6,verify_via_http:[4,6],version:[4,5,6],versionmismatcherror:5,via:[4,6],virtual:2,visibl:6,vital:5,wai:[2,4,6],wait:[2,4],want:[4,6],war:5,warn:6,wea:5,weapon:5,web:6,weblog:6,well:[2,3,4],were:[4,5],weren:4,what:[2,4,5,6],whatev:[4,5],when:[2,4,5,6],whenev:4,where:[2,3,4,5,6],whether:[5,6],which:[2,4,5,6],white:4,white_space_r:4,whitespac:[4,5,6],who:[4,5],whoever:4,whole:6,whose:[2,6],why:4,wiki:6,wikipedia:[5,6],with_sourc:6,within:[2,4,5,6],without:[4,5,6],word:[5,6],work:[0,2,3],work_spec:[4,6],work_spec_nam:4,work_unit:[4,6],work_unit_data:4,work_unit_kei:4,work_unit_key_v:4,work_unit_nam:4,worker_class:4,worker_heartbeat:4,worker_id:4,worker_regist:4,worker_unregist:4,working_set:4,workitem:4,workunit:[4,6],worst:4,would:[2,4,5],wrapper:[4,6],write:[4,6],write_msg_impl:5,write_to:6,writer:[3,4],written:[2,4,6],wrote:[2,5],www:5,x86_64:6,xhtml:5,xml:[5,6],xpath:5,xpath_offset:6,yakonfig:[0,2,4,6],yaml:[2,3,4,6],yaml_files_list:[3,6],year:5,yet:4,yield:[2,4,6],you:[2,3,4,6],your:[2,3,4,6],zero:5,zeu:5,zulu_timestamp:5},titles:["Welcome to StreamCorpus","5. <code class=\"docutils literal\"><span class=\"pre\">dblogger</span></code> &#8212; Python logging setup and database backend","6. <code class=\"docutils literal\"><span class=\"pre\">kvlayer</span></code> &#8212; database abstraction for key/value stores","1. Overview of StreamCorpus software components","7. <code class=\"docutils literal\"><span class=\"pre\">rejester</span></code> &#8212; Redis-based distributed work manager","2. <code class=\"docutils literal\"><span class=\"pre\">streamcorpus</span></code> &#8212; single-document data representation","3. <code class=\"docutils literal\"><span class=\"pre\">streamcorpus_pipeline</span></code> &#8212; stream item processing framework","4. <code class=\"docutils literal\"><span class=\"pre\">yakonfig</span></code> &#8212; global configuration management tools"],titleterms:{"abstract":2,accumulo:2,api:[2,4,6],backend:[1,2],base:4,batch:6,cassandra:2,compon:3,configur:[4,6,7],core:4,data:5,databas:[1,2],dblogger:1,depend:3,detail:4,distribut:4,document:5,exampl:[3,4],except:4,execut:6,filestorag:2,filter:4,framework:6,global:7,implement:4,increment:6,indic:0,integr:6,invoc:3,item:[5,6],kei:2,kvlayer:2,local:2,log:1,manag:[4,7],modul:3,overview:3,pipelin:6,postgr:2,postgrest:2,prioriti:4,process:6,python:1,queue:4,reader:6,redi:[2,4],rejest:[4,6],rejester_work:4,represent:5,riak:2,setup:1,simpl:4,singl:5,softwar:3,split_s3:2,stage:[4,6],storag:5,store:2,stream:[5,6],streamcorpu:[0,3,4,5],streamcorpus_directori:6,streamcorpus_dump:5,streamcorpus_pipelin:6,system:4,tabl:0,task:4,todo:6,tool:[4,5,7],transform:6,valu:2,welcom:0,work:4,worker:4,writer:6,yakonfig:7}})