-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathsearch.js
46 lines (43 loc) · 77.5 KB
/
search.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
window.pdocSearch = (function(){
/** elasticlunr - http://weixsong.github.io * Copyright (C) 2017 Oliver Nightingale * Copyright (C) 2017 Wei Song * MIT Licensed */!function(){function e(e){if(null===e||"object"!=typeof e)return e;var t=e.constructor();for(var n in e)e.hasOwnProperty(n)&&(t[n]=e[n]);return t}var t=function(e){var n=new t.Index;return n.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),e&&e.call(n,n),n};t.version="0.9.5",lunr=t,t.utils={},t.utils.warn=function(e){return function(t){e.console&&console.warn&&console.warn(t)}}(this),t.utils.toString=function(e){return void 0===e||null===e?"":e.toString()},t.EventEmitter=function(){this.events={}},t.EventEmitter.prototype.addListener=function(){var e=Array.prototype.slice.call(arguments),t=e.pop(),n=e;if("function"!=typeof t)throw new TypeError("last argument must be a function");n.forEach(function(e){this.hasHandler(e)||(this.events[e]=[]),this.events[e].push(t)},this)},t.EventEmitter.prototype.removeListener=function(e,t){if(this.hasHandler(e)){var n=this.events[e].indexOf(t);-1!==n&&(this.events[e].splice(n,1),0==this.events[e].length&&delete this.events[e])}},t.EventEmitter.prototype.emit=function(e){if(this.hasHandler(e)){var t=Array.prototype.slice.call(arguments,1);this.events[e].forEach(function(e){e.apply(void 0,t)},this)}},t.EventEmitter.prototype.hasHandler=function(e){return e in this.events},t.tokenizer=function(e){if(!arguments.length||null===e||void 0===e)return[];if(Array.isArray(e)){var n=e.filter(function(e){return null===e||void 0===e?!1:!0});n=n.map(function(e){return t.utils.toString(e).toLowerCase()});var i=[];return n.forEach(function(e){var n=e.split(t.tokenizer.seperator);i=i.concat(n)},this),i}return e.toString().trim().toLowerCase().split(t.tokenizer.seperator)},t.tokenizer.defaultSeperator=/[\s\-]+/,t.tokenizer.seperator=t.tokenizer.defaultSeperator,t.tokenizer.setSeperator=function(e){null!==e&&void 0!==e&&"object"==typeof e&&(t.tokenizer.seperator=e)},t.tokenizer.resetSeperator=function(){t.tokenizer.seperator=t.tokenizer.defaultSeperator},t.tokenizer.getSeperator=function(){return t.tokenizer.seperator},t.Pipeline=function(){this._queue=[]},t.Pipeline.registeredFunctions={},t.Pipeline.registerFunction=function(e,n){n in t.Pipeline.registeredFunctions&&t.utils.warn("Overwriting existing registered function: "+n),e.label=n,t.Pipeline.registeredFunctions[n]=e},t.Pipeline.getRegisteredFunction=function(e){return e in t.Pipeline.registeredFunctions!=!0?null:t.Pipeline.registeredFunctions[e]},t.Pipeline.warnIfFunctionNotRegistered=function(e){var n=e.label&&e.label in this.registeredFunctions;n||t.utils.warn("Function is not registered with pipeline. This may cause problems when serialising the index.\n",e)},t.Pipeline.load=function(e){var n=new t.Pipeline;return e.forEach(function(e){var i=t.Pipeline.getRegisteredFunction(e);if(!i)throw new Error("Cannot load un-registered function: "+e);n.add(i)}),n},t.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach(function(e){t.Pipeline.warnIfFunctionNotRegistered(e),this._queue.push(e)},this)},t.Pipeline.prototype.after=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._queue.indexOf(e);if(-1===i)throw new Error("Cannot find existingFn");this._queue.splice(i+1,0,n)},t.Pipeline.prototype.before=function(e,n){t.Pipeline.warnIfFunctionNotRegistered(n);var i=this._queue.indexOf(e);if(-1===i)throw new Error("Cannot find existingFn");this._queue.splice(i,0,n)},t.Pipeline.prototype.remove=function(e){var t=this._queue.indexOf(e);-1!==t&&this._queue.splice(t,1)},t.Pipeline.prototype.run=function(e){for(var t=[],n=e.length,i=this._queue.length,o=0;n>o;o++){for(var r=e[o],s=0;i>s&&(r=this._queue[s](r,o,e),void 0!==r&&null!==r);s++);void 0!==r&&null!==r&&t.push(r)}return t},t.Pipeline.prototype.reset=function(){this._queue=[]},t.Pipeline.prototype.get=function(){return this._queue},t.Pipeline.prototype.toJSON=function(){return this._queue.map(function(e){return t.Pipeline.warnIfFunctionNotRegistered(e),e.label})},t.Index=function(){this._fields=[],this._ref="id",this.pipeline=new t.Pipeline,this.documentStore=new t.DocumentStore,this.index={},this.eventEmitter=new t.EventEmitter,this._idfCache={},this.on("add","remove","update",function(){this._idfCache={}}.bind(this))},t.Index.prototype.on=function(){var e=Array.prototype.slice.call(arguments);return this.eventEmitter.addListener.apply(this.eventEmitter,e)},t.Index.prototype.off=function(e,t){return this.eventEmitter.removeListener(e,t)},t.Index.load=function(e){e.version!==t.version&&t.utils.warn("version mismatch: current "+t.version+" importing "+e.version);var n=new this;n._fields=e.fields,n._ref=e.ref,n.documentStore=t.DocumentStore.load(e.documentStore),n.pipeline=t.Pipeline.load(e.pipeline),n.index={};for(var i in e.index)n.index[i]=t.InvertedIndex.load(e.index[i]);return n},t.Index.prototype.addField=function(e){return this._fields.push(e),this.index[e]=new t.InvertedIndex,this},t.Index.prototype.setRef=function(e){return this._ref=e,this},t.Index.prototype.saveDocument=function(e){return this.documentStore=new t.DocumentStore(e),this},t.Index.prototype.addDoc=function(e,n){if(e){var n=void 0===n?!0:n,i=e[this._ref];this.documentStore.addDoc(i,e),this._fields.forEach(function(n){var o=this.pipeline.run(t.tokenizer(e[n]));this.documentStore.addFieldLength(i,n,o.length);var r={};o.forEach(function(e){e in r?r[e]+=1:r[e]=1},this);for(var s in r){var u=r[s];u=Math.sqrt(u),this.index[n].addToken(s,{ref:i,tf:u})}},this),n&&this.eventEmitter.emit("add",e,this)}},t.Index.prototype.removeDocByRef=function(e){if(e&&this.documentStore.isDocStored()!==!1&&this.documentStore.hasDoc(e)){var t=this.documentStore.getDoc(e);this.removeDoc(t,!1)}},t.Index.prototype.removeDoc=function(e,n){if(e){var n=void 0===n?!0:n,i=e[this._ref];this.documentStore.hasDoc(i)&&(this.documentStore.removeDoc(i),this._fields.forEach(function(n){var o=this.pipeline.run(t.tokenizer(e[n]));o.forEach(function(e){this.index[n].removeToken(e,i)},this)},this),n&&this.eventEmitter.emit("remove",e,this))}},t.Index.prototype.updateDoc=function(e,t){var t=void 0===t?!0:t;this.removeDocByRef(e[this._ref],!1),this.addDoc(e,!1),t&&this.eventEmitter.emit("update",e,this)},t.Index.prototype.idf=function(e,t){var n="@"+t+"/"+e;if(Object.prototype.hasOwnProperty.call(this._idfCache,n))return this._idfCache[n];var i=this.index[t].getDocFreq(e),o=1+Math.log(this.documentStore.length/(i+1));return this._idfCache[n]=o,o},t.Index.prototype.getFields=function(){return this._fields.slice()},t.Index.prototype.search=function(e,n){if(!e)return[];e="string"==typeof e?{any:e}:JSON.parse(JSON.stringify(e));var i=null;null!=n&&(i=JSON.stringify(n));for(var o=new t.Configuration(i,this.getFields()).get(),r={},s=Object.keys(e),u=0;u<s.length;u++){var a=s[u];r[a]=this.pipeline.run(t.tokenizer(e[a]))}var l={};for(var c in o){var d=r[c]||r.any;if(d){var f=this.fieldSearch(d,c,o),h=o[c].boost;for(var p in f)f[p]=f[p]*h;for(var p in f)p in l?l[p]+=f[p]:l[p]=f[p]}}var v,g=[];for(var p in l)v={ref:p,score:l[p]},this.documentStore.hasDoc(p)&&(v.doc=this.documentStore.getDoc(p)),g.push(v);return g.sort(function(e,t){return t.score-e.score}),g},t.Index.prototype.fieldSearch=function(e,t,n){var i=n[t].bool,o=n[t].expand,r=n[t].boost,s=null,u={};return 0!==r?(e.forEach(function(e){var n=[e];1==o&&(n=this.index[t].expandToken(e));var r={};n.forEach(function(n){var o=this.index[t].getDocs(n),a=this.idf(n,t);if(s&&"AND"==i){var l={};for(var c in s)c in o&&(l[c]=o[c]);o=l}n==e&&this.fieldSearchStats(u,n,o);for(var c in o){var d=this.index[t].getTermFrequency(n,c),f=this.documentStore.getFieldLength(c,t),h=1;0!=f&&(h=1/Math.sqrt(f));var p=1;n!=e&&(p=.15*(1-(n.length-e.length)/n.length));var v=d*a*h*p;c in r?r[c]+=v:r[c]=v}},this),s=this.mergeScores(s,r,i)},this),s=this.coordNorm(s,u,e.length)):void 0},t.Index.prototype.mergeScores=function(e,t,n){if(!e)return t;if("AND"==n){var i={};for(var o in t)o in e&&(i[o]=e[o]+t[o]);return i}for(var o in t)o in e?e[o]+=t[o]:e[o]=t[o];return e},t.Index.prototype.fieldSearchStats=function(e,t,n){for(var i in n)i in e?e[i].push(t):e[i]=[t]},t.Index.prototype.coordNorm=function(e,t,n){for(var i in e)if(i in t){var o=t[i].length;e[i]=e[i]*o/n}return e},t.Index.prototype.toJSON=function(){var e={};return this._fields.forEach(function(t){e[t]=this.index[t].toJSON()},this),{version:t.version,fields:this._fields,ref:this._ref,documentStore:this.documentStore.toJSON(),index:e,pipeline:this.pipeline.toJSON()}},t.Index.prototype.use=function(e){var t=Array.prototype.slice.call(arguments,1);t.unshift(this),e.apply(this,t)},t.DocumentStore=function(e){this._save=null===e||void 0===e?!0:e,this.docs={},this.docInfo={},this.length=0},t.DocumentStore.load=function(e){var t=new this;return t.length=e.length,t.docs=e.docs,t.docInfo=e.docInfo,t._save=e.save,t},t.DocumentStore.prototype.isDocStored=function(){return this._save},t.DocumentStore.prototype.addDoc=function(t,n){this.hasDoc(t)||this.length++,this.docs[t]=this._save===!0?e(n):null},t.DocumentStore.prototype.getDoc=function(e){return this.hasDoc(e)===!1?null:this.docs[e]},t.DocumentStore.prototype.hasDoc=function(e){return e in this.docs},t.DocumentStore.prototype.removeDoc=function(e){this.hasDoc(e)&&(delete this.docs[e],delete this.docInfo[e],this.length--)},t.DocumentStore.prototype.addFieldLength=function(e,t,n){null!==e&&void 0!==e&&0!=this.hasDoc(e)&&(this.docInfo[e]||(this.docInfo[e]={}),this.docInfo[e][t]=n)},t.DocumentStore.prototype.updateFieldLength=function(e,t,n){null!==e&&void 0!==e&&0!=this.hasDoc(e)&&this.addFieldLength(e,t,n)},t.DocumentStore.prototype.getFieldLength=function(e,t){return null===e||void 0===e?0:e in this.docs&&t in this.docInfo[e]?this.docInfo[e][t]:0},t.DocumentStore.prototype.toJSON=function(){return{docs:this.docs,docInfo:this.docInfo,length:this.length,save:this._save}},t.stemmer=function(){var e={ational:"ate",tional:"tion",enci:"ence",anci:"ance",izer:"ize",bli:"ble",alli:"al",entli:"ent",eli:"e",ousli:"ous",ization:"ize",ation:"ate",ator:"ate",alism:"al",iveness:"ive",fulness:"ful",ousness:"ous",aliti:"al",iviti:"ive",biliti:"ble",logi:"log"},t={icate:"ic",ative:"",alize:"al",iciti:"ic",ical:"ic",ful:"",ness:""},n="[^aeiou]",i="[aeiouy]",o=n+"[^aeiouy]*",r=i+"[aeiou]*",s="^("+o+")?"+r+o,u="^("+o+")?"+r+o+"("+r+")?$",a="^("+o+")?"+r+o+r+o,l="^("+o+")?"+i,c=new RegExp(s),d=new RegExp(a),f=new RegExp(u),h=new RegExp(l),p=/^(.+?)(ss|i)es$/,v=/^(.+?)([^s])s$/,g=/^(.+?)eed$/,m=/^(.+?)(ed|ing)$/,y=/.$/,S=/(at|bl|iz)$/,x=new RegExp("([^aeiouylsz])\\1$"),w=new RegExp("^"+o+i+"[^aeiouwxy]$"),I=/^(.+?[^aeiou])y$/,b=/^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/,E=/^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/,D=/^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/,F=/^(.+?)(s|t)(ion)$/,_=/^(.+?)e$/,P=/ll$/,k=new RegExp("^"+o+i+"[^aeiouwxy]$"),z=function(n){var i,o,r,s,u,a,l;if(n.length<3)return n;if(r=n.substr(0,1),"y"==r&&(n=r.toUpperCase()+n.substr(1)),s=p,u=v,s.test(n)?n=n.replace(s,"$1$2"):u.test(n)&&(n=n.replace(u,"$1$2")),s=g,u=m,s.test(n)){var z=s.exec(n);s=c,s.test(z[1])&&(s=y,n=n.replace(s,""))}else if(u.test(n)){var z=u.exec(n);i=z[1],u=h,u.test(i)&&(n=i,u=S,a=x,l=w,u.test(n)?n+="e":a.test(n)?(s=y,n=n.replace(s,"")):l.test(n)&&(n+="e"))}if(s=I,s.test(n)){var z=s.exec(n);i=z[1],n=i+"i"}if(s=b,s.test(n)){var z=s.exec(n);i=z[1],o=z[2],s=c,s.test(i)&&(n=i+e[o])}if(s=E,s.test(n)){var z=s.exec(n);i=z[1],o=z[2],s=c,s.test(i)&&(n=i+t[o])}if(s=D,u=F,s.test(n)){var z=s.exec(n);i=z[1],s=d,s.test(i)&&(n=i)}else if(u.test(n)){var z=u.exec(n);i=z[1]+z[2],u=d,u.test(i)&&(n=i)}if(s=_,s.test(n)){var z=s.exec(n);i=z[1],s=d,u=f,a=k,(s.test(i)||u.test(i)&&!a.test(i))&&(n=i)}return s=P,u=d,s.test(n)&&u.test(n)&&(s=y,n=n.replace(s,"")),"y"==r&&(n=r.toLowerCase()+n.substr(1)),n};return z}(),t.Pipeline.registerFunction(t.stemmer,"stemmer"),t.stopWordFilter=function(e){return e&&t.stopWordFilter.stopWords[e]!==!0?e:void 0},t.clearStopWords=function(){t.stopWordFilter.stopWords={}},t.addStopWords=function(e){null!=e&&Array.isArray(e)!==!1&&e.forEach(function(e){t.stopWordFilter.stopWords[e]=!0},this)},t.resetStopWords=function(){t.stopWordFilter.stopWords=t.defaultStopWords},t.defaultStopWords={"":!0,a:!0,able:!0,about:!0,across:!0,after:!0,all:!0,almost:!0,also:!0,am:!0,among:!0,an:!0,and:!0,any:!0,are:!0,as:!0,at:!0,be:!0,because:!0,been:!0,but:!0,by:!0,can:!0,cannot:!0,could:!0,dear:!0,did:!0,"do":!0,does:!0,either:!0,"else":!0,ever:!0,every:!0,"for":!0,from:!0,get:!0,got:!0,had:!0,has:!0,have:!0,he:!0,her:!0,hers:!0,him:!0,his:!0,how:!0,however:!0,i:!0,"if":!0,"in":!0,into:!0,is:!0,it:!0,its:!0,just:!0,least:!0,let:!0,like:!0,likely:!0,may:!0,me:!0,might:!0,most:!0,must:!0,my:!0,neither:!0,no:!0,nor:!0,not:!0,of:!0,off:!0,often:!0,on:!0,only:!0,or:!0,other:!0,our:!0,own:!0,rather:!0,said:!0,say:!0,says:!0,she:!0,should:!0,since:!0,so:!0,some:!0,than:!0,that:!0,the:!0,their:!0,them:!0,then:!0,there:!0,these:!0,they:!0,"this":!0,tis:!0,to:!0,too:!0,twas:!0,us:!0,wants:!0,was:!0,we:!0,were:!0,what:!0,when:!0,where:!0,which:!0,"while":!0,who:!0,whom:!0,why:!0,will:!0,"with":!0,would:!0,yet:!0,you:!0,your:!0},t.stopWordFilter.stopWords=t.defaultStopWords,t.Pipeline.registerFunction(t.stopWordFilter,"stopWordFilter"),t.trimmer=function(e){if(null===e||void 0===e)throw new Error("token should not be undefined");return e.replace(/^\W+/,"").replace(/\W+$/,"")},t.Pipeline.registerFunction(t.trimmer,"trimmer"),t.InvertedIndex=function(){this.root={docs:{},df:0}},t.InvertedIndex.load=function(e){var t=new this;return t.root=e.root,t},t.InvertedIndex.prototype.addToken=function(e,t,n){for(var n=n||this.root,i=0;i<=e.length-1;){var o=e[i];o in n||(n[o]={docs:{},df:0}),i+=1,n=n[o]}var r=t.ref;n.docs[r]?n.docs[r]={tf:t.tf}:(n.docs[r]={tf:t.tf},n.df+=1)},t.InvertedIndex.prototype.hasToken=function(e){if(!e)return!1;for(var t=this.root,n=0;n<e.length;n++){if(!t[e[n]])return!1;t=t[e[n]]}return!0},t.InvertedIndex.prototype.getNode=function(e){if(!e)return null;for(var t=this.root,n=0;n<e.length;n++){if(!t[e[n]])return null;t=t[e[n]]}return t},t.InvertedIndex.prototype.getDocs=function(e){var t=this.getNode(e);return null==t?{}:t.docs},t.InvertedIndex.prototype.getTermFrequency=function(e,t){var n=this.getNode(e);return null==n?0:t in n.docs?n.docs[t].tf:0},t.InvertedIndex.prototype.getDocFreq=function(e){var t=this.getNode(e);return null==t?0:t.df},t.InvertedIndex.prototype.removeToken=function(e,t){if(e){var n=this.getNode(e);null!=n&&t in n.docs&&(delete n.docs[t],n.df-=1)}},t.InvertedIndex.prototype.expandToken=function(e,t,n){if(null==e||""==e)return[];var t=t||[];if(void 0==n&&(n=this.getNode(e),null==n))return t;n.df>0&&t.push(e);for(var i in n)"docs"!==i&&"df"!==i&&this.expandToken(e+i,t,n[i]);return t},t.InvertedIndex.prototype.toJSON=function(){return{root:this.root}},t.Configuration=function(e,n){var e=e||"";if(void 0==n||null==n)throw new Error("fields should not be null");this.config={};var i;try{i=JSON.parse(e),this.buildUserConfig(i,n)}catch(o){t.utils.warn("user configuration parse failed, will use default configuration"),this.buildDefaultConfig(n)}},t.Configuration.prototype.buildDefaultConfig=function(e){this.reset(),e.forEach(function(e){this.config[e]={boost:1,bool:"OR",expand:!1}},this)},t.Configuration.prototype.buildUserConfig=function(e,n){var i="OR",o=!1;if(this.reset(),"bool"in e&&(i=e.bool||i),"expand"in e&&(o=e.expand||o),"fields"in e)for(var r in e.fields)if(n.indexOf(r)>-1){var s=e.fields[r],u=o;void 0!=s.expand&&(u=s.expand),this.config[r]={boost:s.boost||0===s.boost?s.boost:1,bool:s.bool||i,expand:u}}else t.utils.warn("field name in user configuration not found in index instance fields");else this.addAllFields2UserConfig(i,o,n)},t.Configuration.prototype.addAllFields2UserConfig=function(e,t,n){n.forEach(function(n){this.config[n]={boost:1,bool:e,expand:t}},this)},t.Configuration.prototype.get=function(){return this.config},t.Configuration.prototype.reset=function(){this.config={}},lunr.SortedSet=function(){this.length=0,this.elements=[]},lunr.SortedSet.load=function(e){var t=new this;return t.elements=e,t.length=e.length,t},lunr.SortedSet.prototype.add=function(){var e,t;for(e=0;e<arguments.length;e++)t=arguments[e],~this.indexOf(t)||this.elements.splice(this.locationFor(t),0,t);this.length=this.elements.length},lunr.SortedSet.prototype.toArray=function(){return this.elements.slice()},lunr.SortedSet.prototype.map=function(e,t){return this.elements.map(e,t)},lunr.SortedSet.prototype.forEach=function(e,t){return this.elements.forEach(e,t)},lunr.SortedSet.prototype.indexOf=function(e){for(var t=0,n=this.elements.length,i=n-t,o=t+Math.floor(i/2),r=this.elements[o];i>1;){if(r===e)return o;e>r&&(t=o),r>e&&(n=o),i=n-t,o=t+Math.floor(i/2),r=this.elements[o]}return r===e?o:-1},lunr.SortedSet.prototype.locationFor=function(e){for(var t=0,n=this.elements.length,i=n-t,o=t+Math.floor(i/2),r=this.elements[o];i>1;)e>r&&(t=o),r>e&&(n=o),i=n-t,o=t+Math.floor(i/2),r=this.elements[o];return r>e?o:e>r?o+1:void 0},lunr.SortedSet.prototype.intersect=function(e){for(var t=new lunr.SortedSet,n=0,i=0,o=this.length,r=e.length,s=this.elements,u=e.elements;;){if(n>o-1||i>r-1)break;s[n]!==u[i]?s[n]<u[i]?n++:s[n]>u[i]&&i++:(t.add(s[n]),n++,i++)}return t},lunr.SortedSet.prototype.clone=function(){var e=new lunr.SortedSet;return e.elements=this.toArray(),e.length=e.elements.length,e},lunr.SortedSet.prototype.union=function(e){var t,n,i;this.length>=e.length?(t=this,n=e):(t=e,n=this),i=t.clone();for(var o=0,r=n.toArray();o<r.length;o++)i.add(r[o]);return i},lunr.SortedSet.prototype.toJSON=function(){return this.toArray()},function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():e.elasticlunr=t()}(this,function(){return t})}();
/** pdoc search index */const docs = [{"fullname": "ntv-pandas.docs.user_guide", "modulename": "ntv-pandas.docs.user_guide", "kind": "module", "doc": "<p>Created on Fri Sep 30 10:09:01 2022\n@author: philippe@loco-labs.io</p>\n\n<h3 id=\"ntv-pandas-a-tabular-analyzer-and-a-semantic-compact-and-reversible-converter\"><em>NTV-pandas : A tabular analyzer and a semantic, compact and reversible converter</em></h3>\n\n<p><img src=\"https://loco-philippe.github.io/ES/ntv_pandas.png\" alt=\"ntv-pandas\" align=\"middle\" style=\"height:80px;\"></p>\n\n<p>For more information, see the <a href=\"https://loco-philippe.github.io/ntv-pandas/docs/user_guide.html\">user guide</a> or the <a href=\"https://github.com/loco-philippe/ntv-pandas\">github repository</a>.</p>\n\n<p>NTV-pandas is referenced in the <a href=\"https://pandas.pydata.org/community/ecosystem.html\">pandas ecosystem</a>.</p>\n\n<h1 id=\"why-a-ntv-pandas-converter\">Why a NTV-pandas converter ?</h1>\n\n<p>pandas provide IO converters but limitations are present:</p>\n\n<ul>\n<li>the multidimensional structure is not taken into account,</li>\n<li>the converters are not always reversible (conversion round trip),</li>\n<li>the converters take into account few data types,</li>\n<li>external data types (e.g. TableSchema types) are not included.</li>\n</ul>\n\n<p>pandas does not have a tool for analyzing tabular structures and detecting integrity errors</p>\n\n<h2 id=\"main-features\">main features</h2>\n\n<p>The converter integrates:</p>\n\n<ul>\n<li>interfaces with Xarray, scipp, JSON,</li>\n<li>all the pandas <code>dtype</code> and the data-type associated to a JSON representation,</li>\n<li>an always reversible conversion,</li>\n<li>an identification of tabular and multidimensional structure,</li>\n<li>a full compatibility with <a href=\"http://dataprotocols.org/json-table-schema/#field-types-and-formats\">Table Schema specification</a>.</li>\n</ul>\n\n<p>The NTV-pandas converter uses the <a href=\"https://loco-philippe.github.io/ES/JSON%20semantic%20format%20(JSON-NTV).htm\">semantic NTV format</a>\nto include a large set of data types in a JSON representation.</p>\n\n<p>The NTV-pandas analyzer uses the <a href=\"https://github.com/loco-philippe/tab-analysis/blob/main/README.md\">TAB-analysis</a> tool to analyze and measure the relationships between Fields in DataFrame and the <a href=\"https://github.com/loco-philippe/tab-dataset/blob/main/README.md\">TAB-dataset</a> to identify integrity errors (<a href=\"https://github.com/loco-philippe/ntv-pandas/tree/main/example#readme\">example</a>).</p>\n\n<p>The multidimensional converter uses the <a href=\"https://github.com/loco-philippe/ntv-numpy/blob/main/README.md\">NTV-numpy</a> multidimensional format and interfaces.</p>\n\n<p>NTV-pandas was developped originally in the <a href=\"https://github.com/loco-philippe/NTV\">NTV project</a></p>\n\n<h2 id=\"multidimensional-converter-example\">multidimensional converter example</h2>\n\n<p>In the example below, a Dataframe is converted to Xarray and scipp.</p>\n\n<p>The DataFrame resulting from these conversions are identical to the initial DataFrame (reversibility).</p>\n\n<div class=\"pdoc-code codehilite\">\n<pre><span></span><code><span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">1</span><span class=\"p\">]:</span> <span class=\"kn\">import</span> <span class=\"nn\">pandas</span> <span class=\"k\">as</span> <span class=\"nn\">pd</span>\n <span class=\"kn\">import</span> <span class=\"nn\">ntv_pandas</span> <span class=\"k\">as</span> <span class=\"nn\">npd</span>\n\n<span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">2</span><span class=\"p\">]:</span> <span class=\"n\">fruits</span> <span class=\"o\">=</span> <span class=\"p\">{</span><span class=\"s1\">'plants'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s1\">'fruit'</span><span class=\"p\">,</span> <span class=\"s1\">'fruit'</span><span class=\"p\">,</span> <span class=\"s1\">'fruit'</span><span class=\"p\">,</span> <span class=\"s1\">'fruit'</span><span class=\"p\">,</span> <span class=\"s1\">'vegetable'</span><span class=\"p\">,</span> <span class=\"s1\">'vegetable'</span><span class=\"p\">,</span> <span class=\"s1\">'vegetable'</span><span class=\"p\">,</span> <span class=\"s1\">'vegetable'</span><span class=\"p\">],</span>\n <span class=\"s1\">'plts'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s1\">'fr'</span><span class=\"p\">,</span> <span class=\"s1\">'fr'</span><span class=\"p\">,</span> <span class=\"s1\">'fr'</span><span class=\"p\">,</span> <span class=\"s1\">'fr'</span><span class=\"p\">,</span> <span class=\"s1\">'ve'</span><span class=\"p\">,</span> <span class=\"s1\">'ve'</span><span class=\"p\">,</span> <span class=\"s1\">'ve'</span><span class=\"p\">,</span> <span class=\"s1\">'ve'</span><span class=\"p\">],</span>\n <span class=\"s1\">'quantity'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s1\">'1 kg'</span><span class=\"p\">,</span> <span class=\"s1\">'10 kg'</span><span class=\"p\">,</span> <span class=\"s1\">'1 kg'</span><span class=\"p\">,</span> <span class=\"s1\">'10 kg'</span><span class=\"p\">,</span> <span class=\"s1\">'1 kg'</span><span class=\"p\">,</span> <span class=\"s1\">'10 kg'</span><span class=\"p\">,</span> <span class=\"s1\">'1 kg'</span><span class=\"p\">,</span> <span class=\"s1\">'10 kg'</span><span class=\"p\">],</span>\n <span class=\"s1\">'product'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s1\">'apple'</span><span class=\"p\">,</span> <span class=\"s1\">'apple'</span><span class=\"p\">,</span> <span class=\"s1\">'orange'</span><span class=\"p\">,</span> <span class=\"s1\">'orange'</span><span class=\"p\">,</span> <span class=\"s1\">'peppers'</span><span class=\"p\">,</span> <span class=\"s1\">'peppers'</span><span class=\"p\">,</span> <span class=\"s1\">'carrot'</span><span class=\"p\">,</span> <span class=\"s1\">'carrot'</span><span class=\"p\">],</span>\n <span class=\"s1\">'price'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"mi\">1</span><span class=\"p\">,</span> <span class=\"mi\">10</span><span class=\"p\">,</span> <span class=\"mi\">2</span><span class=\"p\">,</span> <span class=\"mi\">20</span><span class=\"p\">,</span> <span class=\"mf\">1.5</span><span class=\"p\">,</span> <span class=\"mi\">15</span><span class=\"p\">,</span> <span class=\"mf\">1.5</span><span class=\"p\">,</span> <span class=\"mi\">20</span><span class=\"p\">],</span>\n <span class=\"s1\">'price level'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s1\">'low'</span><span class=\"p\">,</span> <span class=\"s1\">'low'</span><span class=\"p\">,</span> <span class=\"s1\">'high'</span><span class=\"p\">,</span> <span class=\"s1\">'high'</span><span class=\"p\">,</span> <span class=\"s1\">'low'</span><span class=\"p\">,</span> <span class=\"s1\">'low'</span><span class=\"p\">,</span> <span class=\"s1\">'high'</span><span class=\"p\">,</span> <span class=\"s1\">'high'</span><span class=\"p\">],</span>\n <span class=\"s1\">'group'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s1\">'fruit 1'</span><span class=\"p\">,</span> <span class=\"s1\">'fruit 10'</span><span class=\"p\">,</span> <span class=\"s1\">'fruit 1'</span><span class=\"p\">,</span> <span class=\"s1\">'veget'</span><span class=\"p\">,</span> <span class=\"s1\">'veget'</span><span class=\"p\">,</span> <span class=\"s1\">'veget'</span><span class=\"p\">,</span> <span class=\"s1\">'veget'</span><span class=\"p\">,</span> <span class=\"s1\">'veget'</span><span class=\"p\">],</span>\n <span class=\"s1\">'id'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"mi\">1001</span><span class=\"p\">,</span> <span class=\"mi\">1002</span><span class=\"p\">,</span> <span class=\"mi\">1003</span><span class=\"p\">,</span> <span class=\"mi\">1004</span><span class=\"p\">,</span> <span class=\"mi\">1005</span><span class=\"p\">,</span> <span class=\"mi\">1006</span><span class=\"p\">,</span> <span class=\"mi\">1007</span><span class=\"p\">,</span> <span class=\"mi\">1008</span><span class=\"p\">],</span>\n <span class=\"s1\">'supplier'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s2\">"sup1"</span><span class=\"p\">,</span> <span class=\"s2\">"sup1"</span><span class=\"p\">,</span> <span class=\"s2\">"sup1"</span><span class=\"p\">,</span> <span class=\"s2\">"sup2"</span><span class=\"p\">,</span> <span class=\"s2\">"sup2"</span><span class=\"p\">,</span> <span class=\"s2\">"sup2"</span><span class=\"p\">,</span> <span class=\"s2\">"sup2"</span><span class=\"p\">,</span> <span class=\"s2\">"sup1"</span><span class=\"p\">],</span>\n <span class=\"s1\">'location'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s2\">"fr"</span><span class=\"p\">,</span> <span class=\"s2\">"gb"</span><span class=\"p\">,</span> <span class=\"s2\">"es"</span><span class=\"p\">,</span> <span class=\"s2\">"ch"</span><span class=\"p\">,</span> <span class=\"s2\">"gb"</span><span class=\"p\">,</span> <span class=\"s2\">"fr"</span><span class=\"p\">,</span> <span class=\"s2\">"es"</span><span class=\"p\">,</span> <span class=\"s2\">"ch"</span><span class=\"p\">],</span>\n <span class=\"s1\">'valid'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s2\">"ok"</span><span class=\"p\">,</span> <span class=\"s2\">"ok"</span><span class=\"p\">,</span> <span class=\"s2\">"ok"</span><span class=\"p\">,</span> <span class=\"s2\">"ok"</span><span class=\"p\">,</span> <span class=\"s2\">"ok"</span><span class=\"p\">,</span> <span class=\"s2\">"ok"</span><span class=\"p\">,</span> <span class=\"s2\">"ok"</span><span class=\"p\">,</span> <span class=\"s2\">"ok"</span><span class=\"p\">]}</span>\n <span class=\"n\">df_fruits</span> <span class=\"o\">=</span> <span class=\"n\">pd</span><span class=\"o\">.</span><span class=\"n\">DataFrame</span><span class=\"p\">(</span><span class=\"n\">fruits</span><span class=\"p\">)</span>\n <span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">npd</span><span class=\"o\">.</span><span class=\"n\">analysis</span><span class=\"p\">(</span><span class=\"n\">distr</span><span class=\"o\">=</span><span class=\"kc\">True</span><span class=\"p\">)</span><span class=\"o\">.</span><span class=\"n\">partitions</span><span class=\"p\">()</span> <span class=\"c1\"># return the list of partitions (a partition is a list of dimensions)</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">2</span><span class=\"p\">]:</span>\n <span class=\"p\">[[</span><span class=\"s1\">'plants'</span><span class=\"p\">,</span> <span class=\"s1\">'quantity'</span><span class=\"p\">,</span> <span class=\"s1\">'price level'</span><span class=\"p\">],</span>\n <span class=\"p\">[</span><span class=\"s1\">'quantity'</span><span class=\"p\">,</span> <span class=\"s1\">'price level'</span><span class=\"p\">,</span> <span class=\"s1\">'supplier'</span><span class=\"p\">],</span>\n <span class=\"p\">[</span><span class=\"s1\">'plants'</span><span class=\"p\">,</span> <span class=\"s1\">'location'</span><span class=\"p\">],</span>\n <span class=\"p\">[</span><span class=\"s1\">'quantity'</span><span class=\"p\">,</span> <span class=\"s1\">'product'</span><span class=\"p\">],</span>\n <span class=\"p\">[</span><span class=\"s1\">'supplier'</span><span class=\"p\">,</span> <span class=\"s1\">'location'</span><span class=\"p\">],</span>\n <span class=\"p\">[</span><span class=\"s1\">'id'</span><span class=\"p\">]]</span>\n\n<span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">3</span><span class=\"p\">]:</span> <span class=\"n\">kwargs</span> <span class=\"o\">=</span> <span class=\"p\">{</span><span class=\"s1\">'dims'</span><span class=\"p\">:[</span><span class=\"s1\">'product'</span><span class=\"p\">,</span> <span class=\"s1\">'quantity'</span><span class=\"p\">],</span> <span class=\"s1\">'datagroup'</span><span class=\"p\">:</span> <span class=\"kc\">False</span><span class=\"p\">,</span> <span class=\"s1\">'ntv_type'</span><span class=\"p\">:</span> <span class=\"kc\">False</span><span class=\"p\">,</span> <span class=\"s1\">'json_name'</span><span class=\"p\">:</span> <span class=\"kc\">False</span><span class=\"p\">}</span>\n <span class=\"n\">xd_fruits</span> <span class=\"o\">=</span> <span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">npd</span><span class=\"o\">.</span><span class=\"n\">to_xarray</span><span class=\"p\">(</span><span class=\"o\">**</span><span class=\"n\">kwargs</span><span class=\"p\">)</span>\n <span class=\"n\">xd_fruits</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">3</span><span class=\"p\">]:</span>\n <span class=\"o\"><</span><span class=\"n\">xarray</span><span class=\"o\">.</span><span class=\"n\">Dataset</span><span class=\"o\">></span> <span class=\"n\">Size</span><span class=\"p\">:</span> <span class=\"mi\">976</span><span class=\"n\">B</span>\n <span class=\"n\">Dimensions</span><span class=\"p\">:</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">:</span> <span class=\"mi\">4</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">:</span> <span class=\"mi\">2</span><span class=\"p\">)</span>\n <span class=\"n\">Coordinates</span><span class=\"p\">:</span>\n <span class=\"o\">*</span> <span class=\"n\">product</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">)</span> <span class=\"o\"><</span><span class=\"n\">U7</span> <span class=\"mi\">112</span><span class=\"n\">B</span> <span class=\"s1\">'apple'</span> <span class=\"s1\">'carrot'</span> <span class=\"s1\">'orange'</span> <span class=\"s1\">'peppers'</span>\n <span class=\"o\">*</span> <span class=\"n\">quantity</span> <span class=\"p\">(</span><span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"o\"><</span><span class=\"n\">U5</span> <span class=\"mi\">40</span><span class=\"n\">B</span> <span class=\"s1\">'1 kg'</span> <span class=\"s1\">'10 kg'</span>\n <span class=\"n\">plants</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">)</span> <span class=\"o\"><</span><span class=\"n\">U9</span> <span class=\"mi\">144</span><span class=\"n\">B</span> <span class=\"s1\">'fruit'</span> <span class=\"s1\">'vegetable'</span> <span class=\"s1\">'fruit'</span> <span class=\"s1\">'vegetable'</span>\n <span class=\"n\">plts</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">)</span> <span class=\"o\"><</span><span class=\"n\">U2</span> <span class=\"mi\">32</span><span class=\"n\">B</span> <span class=\"s1\">'fr'</span> <span class=\"s1\">'ve'</span> <span class=\"s1\">'fr'</span> <span class=\"s1\">'ve'</span>\n <span class=\"n\">price</span> <span class=\"n\">level</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">)</span> <span class=\"o\"><</span><span class=\"n\">U4</span> <span class=\"mi\">64</span><span class=\"n\">B</span> <span class=\"s1\">'low'</span> <span class=\"s1\">'high'</span> <span class=\"s1\">'high'</span> <span class=\"s1\">'low'</span>\n <span class=\"n\">valid</span> <span class=\"o\"><</span><span class=\"n\">U2</span> <span class=\"mi\">8</span><span class=\"n\">B</span> <span class=\"s1\">'ok'</span>\n <span class=\"n\">Data</span> <span class=\"n\">variables</span><span class=\"p\">:</span>\n <span class=\"n\">group</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"o\"><</span><span class=\"n\">U8</span> <span class=\"mi\">256</span><span class=\"n\">B</span> <span class=\"s1\">'fruit 1'</span> <span class=\"s1\">'fruit 10'</span> <span class=\"o\">...</span> <span class=\"s1\">'veget'</span>\n <span class=\"nb\">id</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"n\">int64</span> <span class=\"mi\">64</span><span class=\"n\">B</span> <span class=\"mi\">1001</span> <span class=\"mi\">1002</span> <span class=\"mi\">1007</span> <span class=\"o\">...</span> <span class=\"mi\">1004</span> <span class=\"mi\">1005</span> <span class=\"mi\">1006</span>\n <span class=\"n\">location</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"o\"><</span><span class=\"n\">U2</span> <span class=\"mi\">64</span><span class=\"n\">B</span> <span class=\"s1\">'fr'</span> <span class=\"s1\">'gb'</span> <span class=\"s1\">'es'</span> <span class=\"o\">...</span> <span class=\"s1\">'ch'</span> <span class=\"s1\">'gb'</span> <span class=\"s1\">'fr'</span>\n <span class=\"n\">price</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"n\">float64</span> <span class=\"mi\">64</span><span class=\"n\">B</span> <span class=\"mf\">1.0</span> <span class=\"mf\">10.0</span> <span class=\"mf\">1.5</span> <span class=\"o\">...</span> <span class=\"mf\">20.0</span> <span class=\"mf\">1.5</span> <span class=\"mf\">15.0</span>\n <span class=\"n\">supplier</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"o\"><</span><span class=\"n\">U4</span> <span class=\"mi\">128</span><span class=\"n\">B</span> <span class=\"s1\">'sup1'</span> <span class=\"s1\">'sup1'</span> <span class=\"o\">...</span> <span class=\"s1\">'sup2'</span> <span class=\"s1\">'sup2'</span>\n\n<span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">4</span><span class=\"p\">]:</span> <span class=\"n\">sc_fruits</span> <span class=\"o\">=</span> <span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">npd</span><span class=\"o\">.</span><span class=\"n\">to_scipp</span><span class=\"p\">(</span><span class=\"o\">**</span><span class=\"n\">kwargs</span><span class=\"p\">)</span>\n <span class=\"n\">sc_fruits</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">4</span><span class=\"p\">]:</span>\n <span class=\"o\"><</span><span class=\"n\">scipp</span><span class=\"o\">.</span><span class=\"n\">Dataset</span><span class=\"o\">></span>\n <span class=\"n\">Dimensions</span><span class=\"p\">:</span> <span class=\"n\">Sizes</span><span class=\"p\">[</span><span class=\"n\">product</span><span class=\"p\">:</span><span class=\"mi\">4</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">:</span><span class=\"mi\">2</span><span class=\"p\">,</span> <span class=\"p\">]</span>\n <span class=\"n\">Coordinates</span><span class=\"p\">:</span>\n <span class=\"o\">*</span> <span class=\"n\">plants</span> <span class=\"n\">string</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">)</span> <span class=\"p\">[</span><span class=\"s2\">"fruit"</span><span class=\"p\">,</span> <span class=\"s2\">"vegetable"</span><span class=\"p\">,</span> <span class=\"s2\">"fruit"</span><span class=\"p\">,</span> <span class=\"s2\">"vegetable"</span><span class=\"p\">]</span>\n <span class=\"o\">*</span> <span class=\"n\">plts</span> <span class=\"n\">string</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">)</span> <span class=\"p\">[</span><span class=\"s2\">"fr"</span><span class=\"p\">,</span> <span class=\"s2\">"ve"</span><span class=\"p\">,</span> <span class=\"s2\">"fr"</span><span class=\"p\">,</span> <span class=\"s2\">"ve"</span><span class=\"p\">]</span>\n <span class=\"o\">*</span> <span class=\"n\">price</span> <span class=\"n\">level</span> <span class=\"n\">string</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">)</span> <span class=\"p\">[</span><span class=\"s2\">"low"</span><span class=\"p\">,</span> <span class=\"s2\">"high"</span><span class=\"p\">,</span> <span class=\"s2\">"high"</span><span class=\"p\">,</span> <span class=\"s2\">"low"</span><span class=\"p\">]</span>\n <span class=\"o\">*</span> <span class=\"n\">product</span> <span class=\"n\">string</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">)</span> <span class=\"p\">[</span><span class=\"s2\">"apple"</span><span class=\"p\">,</span> <span class=\"s2\">"carrot"</span><span class=\"p\">,</span> <span class=\"s2\">"orange"</span><span class=\"p\">,</span> <span class=\"s2\">"peppers"</span><span class=\"p\">]</span>\n <span class=\"o\">*</span> <span class=\"n\">quantity</span> <span class=\"n\">string</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">(</span><span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"p\">[</span><span class=\"s2\">"1 kg"</span><span class=\"p\">,</span> <span class=\"s2\">"10 kg"</span><span class=\"p\">]</span>\n <span class=\"o\">*</span> <span class=\"n\">valid</span> <span class=\"n\">string</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">()</span> <span class=\"s2\">"ok"</span>\n <span class=\"n\">Data</span><span class=\"p\">:</span>\n <span class=\"n\">group</span> <span class=\"n\">string</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"p\">[</span><span class=\"s2\">"fruit 1"</span><span class=\"p\">,</span> <span class=\"s2\">"fruit 10"</span><span class=\"p\">,</span> <span class=\"o\">...</span><span class=\"p\">,</span> <span class=\"s2\">"veget"</span><span class=\"p\">,</span> <span class=\"s2\">"veget"</span><span class=\"p\">]</span>\n <span class=\"nb\">id</span> <span class=\"n\">int64</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"p\">[</span><span class=\"mi\">1001</span><span class=\"p\">,</span> <span class=\"mi\">1002</span><span class=\"p\">,</span> <span class=\"o\">...</span><span class=\"p\">,</span> <span class=\"mi\">1005</span><span class=\"p\">,</span> <span class=\"mi\">1006</span><span class=\"p\">]</span>\n <span class=\"n\">location</span> <span class=\"n\">string</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"p\">[</span><span class=\"s2\">"fr"</span><span class=\"p\">,</span> <span class=\"s2\">"gb"</span><span class=\"p\">,</span> <span class=\"o\">...</span><span class=\"p\">,</span> <span class=\"s2\">"gb"</span><span class=\"p\">,</span> <span class=\"s2\">"fr"</span><span class=\"p\">]</span>\n <span class=\"n\">price</span> <span class=\"n\">float64</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"p\">[</span><span class=\"mi\">1</span><span class=\"p\">,</span> <span class=\"mi\">10</span><span class=\"p\">,</span> <span class=\"o\">...</span><span class=\"p\">,</span> <span class=\"mf\">1.5</span><span class=\"p\">,</span> <span class=\"mi\">15</span><span class=\"p\">]</span>\n <span class=\"n\">supplier</span> <span class=\"n\">string</span> <span class=\"p\">[</span><span class=\"n\">dimensionless</span><span class=\"p\">]</span> <span class=\"p\">(</span><span class=\"n\">product</span><span class=\"p\">,</span> <span class=\"n\">quantity</span><span class=\"p\">)</span> <span class=\"p\">[</span><span class=\"s2\">"sup1"</span><span class=\"p\">,</span> <span class=\"s2\">"sup1"</span><span class=\"p\">,</span> <span class=\"o\">...</span><span class=\"p\">,</span> <span class=\"s2\">"sup2"</span><span class=\"p\">,</span> <span class=\"s2\">"sup2"</span><span class=\"p\">]</span>\n</code></pre>\n</div>\n\n<p>Reversibility:</p>\n\n<div class=\"pdoc-code codehilite\">\n<pre><span></span><code><span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">5</span><span class=\"p\">]:</span> <span class=\"n\">df_fruits_xd</span> <span class=\"o\">=</span> <span class=\"n\">npd</span><span class=\"o\">.</span><span class=\"n\">from_xarray</span><span class=\"p\">(</span><span class=\"n\">xd_fruits</span><span class=\"p\">,</span> <span class=\"o\">**</span><span class=\"n\">kwargs</span><span class=\"p\">)</span>\n <span class=\"n\">df_fruits_xd_sort</span> <span class=\"o\">=</span> <span class=\"n\">df_fruits_xd</span><span class=\"o\">.</span><span class=\"n\">reset_index</span><span class=\"p\">()[</span><span class=\"nb\">list</span><span class=\"p\">(</span><span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">columns</span><span class=\"p\">)]</span><span class=\"o\">.</span><span class=\"n\">sort_values</span><span class=\"p\">(</span><span class=\"nb\">list</span><span class=\"p\">(</span><span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">columns</span><span class=\"p\">))</span><span class=\"o\">.</span><span class=\"n\">reset_index</span><span class=\"p\">(</span><span class=\"n\">drop</span><span class=\"o\">=</span><span class=\"kc\">True</span><span class=\"p\">)</span>\n <span class=\"n\">df_fruits_sort</span> <span class=\"o\">=</span> <span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">sort_values</span><span class=\"p\">(</span><span class=\"nb\">list</span><span class=\"p\">(</span><span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">columns</span><span class=\"p\">))</span><span class=\"o\">.</span><span class=\"n\">reset_index</span><span class=\"p\">(</span><span class=\"n\">drop</span><span class=\"o\">=</span><span class=\"kc\">True</span><span class=\"p\">)</span>\n <span class=\"n\">df_fruits_xd_sort</span><span class=\"o\">.</span><span class=\"n\">equals</span><span class=\"p\">(</span><span class=\"n\">df_fruits_sort</span><span class=\"p\">)</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">5</span><span class=\"p\">]:</span>\n <span class=\"kc\">True</span>\n\n<span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">6</span><span class=\"p\">]:</span> <span class=\"n\">df_fruits_sc</span> <span class=\"o\">=</span> <span class=\"n\">npd</span><span class=\"o\">.</span><span class=\"n\">from_scipp</span><span class=\"p\">(</span><span class=\"n\">sc_fruits</span><span class=\"p\">,</span> <span class=\"o\">**</span><span class=\"n\">kwargs</span><span class=\"p\">)</span>\n <span class=\"n\">df_fruits_sc_sort</span> <span class=\"o\">=</span> <span class=\"n\">df_fruits_sc</span><span class=\"o\">.</span><span class=\"n\">reset_index</span><span class=\"p\">()[</span><span class=\"nb\">list</span><span class=\"p\">(</span><span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">columns</span><span class=\"p\">)]</span><span class=\"o\">.</span><span class=\"n\">sort_values</span><span class=\"p\">(</span><span class=\"nb\">list</span><span class=\"p\">(</span><span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">columns</span><span class=\"p\">))</span><span class=\"o\">.</span><span class=\"n\">reset_index</span><span class=\"p\">(</span><span class=\"n\">drop</span><span class=\"o\">=</span><span class=\"kc\">True</span><span class=\"p\">)</span>\n <span class=\"n\">df_fruits_sort</span> <span class=\"o\">=</span> <span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">sort_values</span><span class=\"p\">(</span><span class=\"nb\">list</span><span class=\"p\">(</span><span class=\"n\">df_fruits</span><span class=\"o\">.</span><span class=\"n\">columns</span><span class=\"p\">))</span><span class=\"o\">.</span><span class=\"n\">reset_index</span><span class=\"p\">(</span><span class=\"n\">drop</span><span class=\"o\">=</span><span class=\"kc\">True</span><span class=\"p\">)</span>\n <span class=\"n\">df_fruits_sc_sort</span><span class=\"o\">.</span><span class=\"n\">equals</span><span class=\"p\">(</span><span class=\"n\">df_fruits_sort</span><span class=\"p\">)</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">6</span><span class=\"p\">]:</span>\n <span class=\"kc\">True</span>\n</code></pre>\n</div>\n\n<h2 id=\"json-converter-example\">JSON converter example</h2>\n\n<p>In the example below, a DataFrame with multiple data types is converted to JSON (first to NTV format and then to Table Schema format).</p>\n\n<p>The DataFrame resulting from these JSON conversions are identical to the initial DataFrame (reversibility).</p>\n\n<p>With the existing JSON interface, these conversions are not possible.</p>\n\n<div class=\"pdoc-code codehilite\">\n<pre><span></span><code><span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">1</span><span class=\"p\">]:</span> <span class=\"kn\">from</span> <span class=\"nn\">shapely.geometry</span> <span class=\"kn\">import</span> <span class=\"n\">Point</span>\n <span class=\"kn\">from</span> <span class=\"nn\">datetime</span> <span class=\"kn\">import</span> <span class=\"n\">date</span>\n <span class=\"kn\">import</span> <span class=\"nn\">pandas</span> <span class=\"k\">as</span> <span class=\"nn\">pd</span>\n <span class=\"kn\">import</span> <span class=\"nn\">ntv_pandas</span> <span class=\"k\">as</span> <span class=\"nn\">npd</span>\n\n<span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">2</span><span class=\"p\">]:</span> <span class=\"n\">data</span> <span class=\"o\">=</span> <span class=\"p\">{</span><span class=\"s1\">'index'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"mi\">100</span><span class=\"p\">,</span> <span class=\"mi\">200</span><span class=\"p\">,</span> <span class=\"mi\">300</span><span class=\"p\">,</span> <span class=\"mi\">400</span><span class=\"p\">,</span> <span class=\"mi\">500</span><span class=\"p\">],</span>\n <span class=\"s1\">'dates::date'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"n\">date</span><span class=\"p\">(</span><span class=\"mi\">1964</span><span class=\"p\">,</span><span class=\"mi\">1</span><span class=\"p\">,</span><span class=\"mi\">1</span><span class=\"p\">),</span> <span class=\"n\">date</span><span class=\"p\">(</span><span class=\"mi\">1985</span><span class=\"p\">,</span><span class=\"mi\">2</span><span class=\"p\">,</span><span class=\"mi\">5</span><span class=\"p\">),</span> <span class=\"n\">date</span><span class=\"p\">(</span><span class=\"mi\">2022</span><span class=\"p\">,</span><span class=\"mi\">1</span><span class=\"p\">,</span><span class=\"mi\">21</span><span class=\"p\">),</span> <span class=\"n\">date</span><span class=\"p\">(</span><span class=\"mi\">1964</span><span class=\"p\">,</span><span class=\"mi\">1</span><span class=\"p\">,</span><span class=\"mi\">1</span><span class=\"p\">),</span> <span class=\"n\">date</span><span class=\"p\">(</span><span class=\"mi\">1985</span><span class=\"p\">,</span><span class=\"mi\">2</span><span class=\"p\">,</span><span class=\"mi\">5</span><span class=\"p\">)],</span>\n <span class=\"s1\">'value'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"mi\">10</span><span class=\"p\">,</span> <span class=\"mi\">10</span><span class=\"p\">,</span> <span class=\"mi\">20</span><span class=\"p\">,</span> <span class=\"mi\">20</span><span class=\"p\">,</span> <span class=\"mi\">30</span><span class=\"p\">],</span>\n <span class=\"s1\">'value32'</span><span class=\"p\">:</span> <span class=\"n\">pd</span><span class=\"o\">.</span><span class=\"n\">Series</span><span class=\"p\">([</span><span class=\"mi\">12</span><span class=\"p\">,</span> <span class=\"mi\">12</span><span class=\"p\">,</span> <span class=\"mi\">22</span><span class=\"p\">,</span> <span class=\"mi\">22</span><span class=\"p\">,</span> <span class=\"mi\">32</span><span class=\"p\">],</span> <span class=\"n\">dtype</span><span class=\"o\">=</span><span class=\"s1\">'int32'</span><span class=\"p\">),</span>\n <span class=\"s1\">'res'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"mi\">10</span><span class=\"p\">,</span> <span class=\"mi\">20</span><span class=\"p\">,</span> <span class=\"mi\">30</span><span class=\"p\">,</span> <span class=\"mi\">10</span><span class=\"p\">,</span> <span class=\"mi\">20</span><span class=\"p\">],</span>\n <span class=\"s1\">'coord::point'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"n\">Point</span><span class=\"p\">(</span><span class=\"mi\">1</span><span class=\"p\">,</span><span class=\"mi\">2</span><span class=\"p\">),</span> <span class=\"n\">Point</span><span class=\"p\">(</span><span class=\"mi\">3</span><span class=\"p\">,</span><span class=\"mi\">4</span><span class=\"p\">),</span> <span class=\"n\">Point</span><span class=\"p\">(</span><span class=\"mi\">5</span><span class=\"p\">,</span><span class=\"mi\">6</span><span class=\"p\">),</span> <span class=\"n\">Point</span><span class=\"p\">(</span><span class=\"mi\">7</span><span class=\"p\">,</span><span class=\"mi\">8</span><span class=\"p\">),</span> <span class=\"n\">Point</span><span class=\"p\">(</span><span class=\"mi\">3</span><span class=\"p\">,</span><span class=\"mi\">4</span><span class=\"p\">)],</span>\n <span class=\"s1\">'names'</span><span class=\"p\">:</span> <span class=\"n\">pd</span><span class=\"o\">.</span><span class=\"n\">Series</span><span class=\"p\">([</span><span class=\"s1\">'john'</span><span class=\"p\">,</span> <span class=\"s1\">'eric'</span><span class=\"p\">,</span> <span class=\"s1\">'judith'</span><span class=\"p\">,</span> <span class=\"s1\">'mila'</span><span class=\"p\">,</span> <span class=\"s1\">'hector'</span><span class=\"p\">],</span> <span class=\"n\">dtype</span><span class=\"o\">=</span><span class=\"s1\">'string'</span><span class=\"p\">),</span>\n <span class=\"s1\">'unique'</span><span class=\"p\">:</span> <span class=\"kc\">True</span> <span class=\"p\">}</span>\n\n<span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">3</span><span class=\"p\">]:</span> <span class=\"n\">df</span> <span class=\"o\">=</span> <span class=\"n\">pd</span><span class=\"o\">.</span><span class=\"n\">DataFrame</span><span class=\"p\">(</span><span class=\"n\">data</span><span class=\"p\">)</span><span class=\"o\">.</span><span class=\"n\">set_index</span><span class=\"p\">(</span><span class=\"s1\">'index'</span><span class=\"p\">)</span>\n <span class=\"n\">df</span><span class=\"o\">.</span><span class=\"n\">index</span><span class=\"o\">.</span><span class=\"n\">name</span> <span class=\"o\">=</span> <span class=\"kc\">None</span>\n\n<span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">4</span><span class=\"p\">]:</span> <span class=\"n\">df</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">4</span><span class=\"p\">]:</span> <span class=\"n\">dates</span><span class=\"p\">::</span><span class=\"n\">date</span> <span class=\"n\">value</span> <span class=\"n\">value32</span> <span class=\"n\">res</span> <span class=\"n\">coord</span><span class=\"p\">::</span><span class=\"n\">point</span> <span class=\"n\">names</span> <span class=\"n\">unique</span>\n <span class=\"mi\">100</span> <span class=\"mi\">1964</span><span class=\"o\">-</span><span class=\"mi\">01</span><span class=\"o\">-</span><span class=\"mi\">01</span> <span class=\"mi\">10</span> <span class=\"mi\">12</span> <span class=\"mi\">10</span> <span class=\"n\">POINT</span> <span class=\"p\">(</span><span class=\"mi\">1</span> <span class=\"mi\">2</span><span class=\"p\">)</span> <span class=\"n\">john</span> <span class=\"kc\">True</span>\n <span class=\"mi\">200</span> <span class=\"mi\">1985</span><span class=\"o\">-</span><span class=\"mi\">02</span><span class=\"o\">-</span><span class=\"mi\">05</span> <span class=\"mi\">10</span> <span class=\"mi\">12</span> <span class=\"mi\">20</span> <span class=\"n\">POINT</span> <span class=\"p\">(</span><span class=\"mi\">3</span> <span class=\"mi\">4</span><span class=\"p\">)</span> <span class=\"n\">eric</span> <span class=\"kc\">True</span>\n <span class=\"mi\">300</span> <span class=\"mi\">2022</span><span class=\"o\">-</span><span class=\"mi\">01</span><span class=\"o\">-</span><span class=\"mi\">21</span> <span class=\"mi\">20</span> <span class=\"mi\">22</span> <span class=\"mi\">30</span> <span class=\"n\">POINT</span> <span class=\"p\">(</span><span class=\"mi\">5</span> <span class=\"mi\">6</span><span class=\"p\">)</span> <span class=\"n\">judith</span> <span class=\"kc\">True</span>\n <span class=\"mi\">400</span> <span class=\"mi\">1964</span><span class=\"o\">-</span><span class=\"mi\">01</span><span class=\"o\">-</span><span class=\"mi\">01</span> <span class=\"mi\">20</span> <span class=\"mi\">22</span> <span class=\"mi\">10</span> <span class=\"n\">POINT</span> <span class=\"p\">(</span><span class=\"mi\">7</span> <span class=\"mi\">8</span><span class=\"p\">)</span> <span class=\"n\">mila</span> <span class=\"kc\">True</span>\n <span class=\"mi\">500</span> <span class=\"mi\">1985</span><span class=\"o\">-</span><span class=\"mi\">02</span><span class=\"o\">-</span><span class=\"mi\">05</span> <span class=\"mi\">30</span> <span class=\"mi\">32</span> <span class=\"mi\">20</span> <span class=\"n\">POINT</span> <span class=\"p\">(</span><span class=\"mi\">3</span> <span class=\"mi\">4</span><span class=\"p\">)</span> <span class=\"n\">hector</span> <span class=\"kc\">True</span>\n</code></pre>\n</div>\n\n<p>JSON-NTV representation:</p>\n\n<div class=\"pdoc-code codehilite\">\n<pre><span></span><code><span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">5</span><span class=\"p\">]:</span> <span class=\"n\">df_to_json</span> <span class=\"o\">=</span> <span class=\"n\">df</span><span class=\"o\">.</span><span class=\"n\">npd</span><span class=\"o\">.</span><span class=\"n\">to_json</span><span class=\"p\">()</span>\n <span class=\"n\">pprint</span><span class=\"p\">(</span><span class=\"n\">df_to_json</span><span class=\"p\">,</span> <span class=\"n\">compact</span><span class=\"o\">=</span><span class=\"kc\">True</span><span class=\"p\">,</span> <span class=\"n\">width</span><span class=\"o\">=</span><span class=\"mi\">120</span><span class=\"p\">,</span> <span class=\"n\">sort_dicts</span><span class=\"o\">=</span><span class=\"kc\">False</span><span class=\"p\">)</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">5</span><span class=\"p\">]:</span> <span class=\"p\">{</span><span class=\"s1\">':tab'</span><span class=\"p\">:</span> <span class=\"p\">{</span><span class=\"s1\">'index'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"mi\">100</span><span class=\"p\">,</span> <span class=\"mi\">200</span><span class=\"p\">,</span> <span class=\"mi\">300</span><span class=\"p\">,</span> <span class=\"mi\">400</span><span class=\"p\">,</span> <span class=\"mi\">500</span><span class=\"p\">],</span>\n <span class=\"s1\">'dates::date'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s1\">'1964-01-01'</span><span class=\"p\">,</span> <span class=\"s1\">'1985-02-05'</span><span class=\"p\">,</span> <span class=\"s1\">'2022-01-21'</span><span class=\"p\">,</span> <span class=\"s1\">'1964-01-01'</span><span class=\"p\">,</span> <span class=\"s1\">'1985-02-05'</span><span class=\"p\">],</span>\n <span class=\"s1\">'value'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"mi\">10</span><span class=\"p\">,</span> <span class=\"mi\">10</span><span class=\"p\">,</span> <span class=\"mi\">20</span><span class=\"p\">,</span> <span class=\"mi\">20</span><span class=\"p\">,</span> <span class=\"mi\">30</span><span class=\"p\">],</span>\n <span class=\"s1\">'value32::int32'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"mi\">12</span><span class=\"p\">,</span> <span class=\"mi\">12</span><span class=\"p\">,</span> <span class=\"mi\">22</span><span class=\"p\">,</span> <span class=\"mi\">22</span><span class=\"p\">,</span> <span class=\"mi\">32</span><span class=\"p\">],</span>\n <span class=\"s1\">'res'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"mi\">10</span><span class=\"p\">,</span> <span class=\"mi\">20</span><span class=\"p\">,</span> <span class=\"mi\">30</span><span class=\"p\">,</span> <span class=\"mi\">10</span><span class=\"p\">,</span> <span class=\"mi\">20</span><span class=\"p\">],</span>\n <span class=\"s1\">'coord::point'</span><span class=\"p\">:</span> <span class=\"p\">[[</span><span class=\"mf\">1.0</span><span class=\"p\">,</span> <span class=\"mf\">2.0</span><span class=\"p\">],</span> <span class=\"p\">[</span><span class=\"mf\">3.0</span><span class=\"p\">,</span> <span class=\"mf\">4.0</span><span class=\"p\">],</span> <span class=\"p\">[</span><span class=\"mf\">5.0</span><span class=\"p\">,</span> <span class=\"mf\">6.0</span><span class=\"p\">],</span> <span class=\"p\">[</span><span class=\"mf\">7.0</span><span class=\"p\">,</span> <span class=\"mf\">8.0</span><span class=\"p\">],</span> <span class=\"p\">[</span><span class=\"mf\">3.0</span><span class=\"p\">,</span> <span class=\"mf\">4.0</span><span class=\"p\">]],</span>\n <span class=\"s1\">'names::string'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s1\">'john'</span><span class=\"p\">,</span> <span class=\"s1\">'eric'</span><span class=\"p\">,</span> <span class=\"s1\">'judith'</span><span class=\"p\">,</span> <span class=\"s1\">'mila'</span><span class=\"p\">,</span> <span class=\"s1\">'hector'</span><span class=\"p\">],</span>\n <span class=\"s1\">'unique'</span><span class=\"p\">:</span> <span class=\"kc\">True</span><span class=\"p\">}}</span>\n</code></pre>\n</div>\n\n<p>Reversibility:</p>\n\n<div class=\"pdoc-code codehilite\">\n<pre><span></span><code><span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">6</span><span class=\"p\">]:</span> <span class=\"nb\">print</span><span class=\"p\">(</span><span class=\"n\">npd</span><span class=\"o\">.</span><span class=\"n\">read_json</span><span class=\"p\">(</span><span class=\"n\">df_to_json</span><span class=\"p\">)</span><span class=\"o\">.</span><span class=\"n\">equals</span><span class=\"p\">(</span><span class=\"n\">df</span><span class=\"p\">))</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">6</span><span class=\"p\">]:</span> <span class=\"kc\">True</span>\n</code></pre>\n</div>\n\n<p>Table Schema representation:</p>\n\n<div class=\"pdoc-code codehilite\">\n<pre><span></span><code><span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">7</span><span class=\"p\">]:</span> <span class=\"n\">df_to_table</span> <span class=\"o\">=</span> <span class=\"n\">df</span><span class=\"o\">.</span><span class=\"n\">npd</span><span class=\"o\">.</span><span class=\"n\">to_json</span><span class=\"p\">(</span><span class=\"n\">table</span><span class=\"o\">=</span><span class=\"kc\">True</span><span class=\"p\">)</span>\n <span class=\"n\">pprint</span><span class=\"p\">(</span><span class=\"n\">df_to_table</span><span class=\"p\">[</span><span class=\"s1\">'data'</span><span class=\"p\">][</span><span class=\"mi\">0</span><span class=\"p\">],</span> <span class=\"n\">sort_dicts</span><span class=\"o\">=</span><span class=\"kc\">False</span><span class=\"p\">)</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">7</span><span class=\"p\">]:</span> <span class=\"p\">{</span><span class=\"s1\">'index'</span><span class=\"p\">:</span> <span class=\"mi\">100</span><span class=\"p\">,</span>\n <span class=\"s1\">'dates'</span><span class=\"p\">:</span> <span class=\"s1\">'1964-01-01'</span><span class=\"p\">,</span>\n <span class=\"s1\">'value'</span><span class=\"p\">:</span> <span class=\"mi\">10</span><span class=\"p\">,</span>\n <span class=\"s1\">'value32'</span><span class=\"p\">:</span> <span class=\"mi\">12</span><span class=\"p\">,</span>\n <span class=\"s1\">'res'</span><span class=\"p\">:</span> <span class=\"mi\">10</span><span class=\"p\">,</span>\n <span class=\"s1\">'coord'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"mf\">1.0</span><span class=\"p\">,</span> <span class=\"mf\">2.0</span><span class=\"p\">],</span>\n <span class=\"s1\">'names'</span><span class=\"p\">:</span> <span class=\"s1\">'john'</span><span class=\"p\">,</span>\n <span class=\"s1\">'unique'</span><span class=\"p\">:</span> <span class=\"kc\">True</span><span class=\"p\">}</span>\n\n<span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">8</span><span class=\"p\">]:</span> <span class=\"n\">pprint</span><span class=\"p\">(</span><span class=\"n\">df_to_table</span><span class=\"p\">[</span><span class=\"s1\">'schema'</span><span class=\"p\">],</span> <span class=\"n\">sort_dicts</span><span class=\"o\">=</span><span class=\"kc\">False</span><span class=\"p\">)</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">8</span><span class=\"p\">]:</span> <span class=\"p\">{</span><span class=\"s1\">'fields'</span><span class=\"p\">:</span> <span class=\"p\">[{</span><span class=\"s1\">'name'</span><span class=\"p\">:</span> <span class=\"s1\">'index'</span><span class=\"p\">,</span> <span class=\"s1\">'type'</span><span class=\"p\">:</span> <span class=\"s1\">'integer'</span><span class=\"p\">},</span>\n <span class=\"p\">{</span><span class=\"s1\">'name'</span><span class=\"p\">:</span> <span class=\"s1\">'dates'</span><span class=\"p\">,</span> <span class=\"s1\">'type'</span><span class=\"p\">:</span> <span class=\"s1\">'date'</span><span class=\"p\">},</span>\n <span class=\"p\">{</span><span class=\"s1\">'name'</span><span class=\"p\">:</span> <span class=\"s1\">'value'</span><span class=\"p\">,</span> <span class=\"s1\">'type'</span><span class=\"p\">:</span> <span class=\"s1\">'integer'</span><span class=\"p\">},</span>\n <span class=\"p\">{</span><span class=\"s1\">'name'</span><span class=\"p\">:</span> <span class=\"s1\">'value32'</span><span class=\"p\">,</span> <span class=\"s1\">'type'</span><span class=\"p\">:</span> <span class=\"s1\">'integer'</span><span class=\"p\">,</span> <span class=\"s1\">'format'</span><span class=\"p\">:</span> <span class=\"s1\">'int32'</span><span class=\"p\">},</span>\n <span class=\"p\">{</span><span class=\"s1\">'name'</span><span class=\"p\">:</span> <span class=\"s1\">'res'</span><span class=\"p\">,</span> <span class=\"s1\">'type'</span><span class=\"p\">:</span> <span class=\"s1\">'integer'</span><span class=\"p\">},</span>\n <span class=\"p\">{</span><span class=\"s1\">'name'</span><span class=\"p\">:</span> <span class=\"s1\">'coord'</span><span class=\"p\">,</span> <span class=\"s1\">'type'</span><span class=\"p\">:</span> <span class=\"s1\">'geopoint'</span><span class=\"p\">,</span> <span class=\"s1\">'format'</span><span class=\"p\">:</span> <span class=\"s1\">'array'</span><span class=\"p\">},</span>\n <span class=\"p\">{</span><span class=\"s1\">'name'</span><span class=\"p\">:</span> <span class=\"s1\">'names'</span><span class=\"p\">,</span> <span class=\"s1\">'type'</span><span class=\"p\">:</span> <span class=\"s1\">'string'</span><span class=\"p\">},</span>\n <span class=\"p\">{</span><span class=\"s1\">'name'</span><span class=\"p\">:</span> <span class=\"s1\">'unique'</span><span class=\"p\">,</span> <span class=\"s1\">'type'</span><span class=\"p\">:</span> <span class=\"s1\">'boolean'</span><span class=\"p\">}],</span>\n <span class=\"s1\">'primaryKey'</span><span class=\"p\">:</span> <span class=\"p\">[</span><span class=\"s1\">'index'</span><span class=\"p\">],</span>\n <span class=\"s1\">'pandas_version'</span><span class=\"p\">:</span> <span class=\"s1\">'1.4.0'</span><span class=\"p\">}</span>\n</code></pre>\n</div>\n\n<p>Reversibility:</p>\n\n<div class=\"pdoc-code codehilite\">\n<pre><span></span><code><span class=\"n\">In</span> <span class=\"p\">[</span><span class=\"mi\">9</span><span class=\"p\">]:</span> <span class=\"nb\">print</span><span class=\"p\">(</span><span class=\"n\">npd</span><span class=\"o\">.</span><span class=\"n\">read_json</span><span class=\"p\">(</span><span class=\"n\">df_to_table</span><span class=\"p\">)</span><span class=\"o\">.</span><span class=\"n\">equals</span><span class=\"p\">(</span><span class=\"n\">df</span><span class=\"p\">))</span>\n<span class=\"n\">Out</span><span class=\"p\">[</span><span class=\"mi\">9</span><span class=\"p\">]:</span> <span class=\"kc\">True</span>\n</code></pre>\n</div>\n\n<h1 id=\"installation\">installation</h1>\n\n<h2 id=\"ntv_pandas-package\">ntv_pandas package</h2>\n\n<p>The <code>ntv_pandas</code> package includes</p>\n\n<ul>\n<li><code>pandas_ntv_connector</code> module\n<ul>\n<li>functions <code>read_json</code> and <code>to_json</code> to convert JSON data and pandas entities</li>\n<li>functions <code>check_relation</code> and <code>to_analysis</code></li>\n<li>functions <code>from_xarray</code> and <code>from_scipp</code></li>\n<li>function <code>as_def_type</code> to convert a Series or DataFrame with default <code>dtype</code> and <code>equals</code> to extend pandas <code>equals</code> method</li>\n<li>child classes of <code>NTV.json_ntv.ntv.NtvConnector</code> abstract class:\n<ul>\n<li><code>DataFrameConnec</code>: 'tab' connector</li>\n<li><code>SeriesConnec</code>: 'field' connector</li>\n</ul></li>\n<li>an utility class with static methods : <code>PdUtil</code></li>\n</ul></li>\n<li><code>accessors</code> modules\n<ul>\n<li><code>NpdDataFrameAccessor</code>: DataFrame accessor</li>\n<li><code>NpdSeriesAccessor</code>: Series accessor</li>\n</ul></li>\n<li>configuration files:\n<ul>\n<li><code>ntv_pandas.ini</code> (correspondence between ntv_type and pandas dtype)</li>\n<li><code>ntv_table.ini</code> (correspondence between ntv_type and Table Schema types)</li>\n</ul></li>\n</ul>\n\n<h2 id=\"installation-2\">Installation</h2>\n\n<p><code>ntv_pandas</code> itself is a pure Python package. maintained on <a href=\"https://github.com/loco-philippe/ntv-pandas\">ntv-pandas github repository</a>.</p>\n\n<p>It can be installed with <code>pip</code>.</p>\n\n<pre><code>pip install ntv_pandas\n</code></pre>\n\n<p>dependency:</p>\n\n<ul>\n<li><code>json_ntv</code>: support the NTV format,</li>\n<li><code>shapely</code>: for the location data,</li>\n<li><code>pandas</code></li>\n<li><code>tab_analysis</code>: tabular structure analysis</li>\n<li><code>ntv_numpy</code>: multidimensional format</li>\n</ul>\n\n<h1 id=\"examples-and-uses\">Examples and uses</h1>\n\n<h2 id=\"multidimensional-examples\">Multidimensional examples</h2>\n\n<p>The <a href=\"https://nbviewer.org/github/loco-philippe/ntv-pandas/blob/main/example/example_multidimensional.ipynb\">first notebook</a> presents the multidimensional interfaces.</p>\n\n<p>A <a href=\"https://nbviewer.org/github/loco-philippe/ntv-pandas/blob/main/example/xarray_pandas_converter.ipynb\">second notebook</a> show how the ntv_pandas converter complements the existing Xarray interface.</p>\n\n<p>A <a href=\"https://nbviewer.org/github/loco-philippe/ntv-pandas/blob/main/example/example_xarray.ipynb\">simple use case</a> shows the advantage of multidimensional representation (conversion Xarray of a dataset, optimization of data size).</p>\n\n<h2 id=\"dataframe-integrity\">DataFrame integrity</h2>\n\n<p>The <a href=\"https://nbviewer.org/github/loco-philippe/ntv-pandas/blob/main/example/example_analysis.ipynb\">integrity notebook</a> explains how identify integrity errors in a DataFrame</p>\n\n<h2 id=\"json-format\">JSON format</h2>\n\n<p>The <a href=\"https://nbviewer.org/github/loco-philippe/ntv-pandas/blob/main/example/example_json_pandas.ipynb\">first notebook</a> presents limitations of existing json-pandas interface.</p>\n\n<p>The <a href=\"https://nbviewer.org/github/loco-philippe/ntv-pandas/blob/main/example/example_ntv_pandas.ipynb\">ntv-pandas</a> notebook presents the JSON interface and the <a href=\"https://nbviewer.org/github/loco-philippe/ntv-pandas/blob/main/example/example_table_pandas.ipynb\">table-pandas</a> notebook presents the table-schema interface.</p>\n\n<h1 id=\"documentation\">documentation</h1>\n\n<p>The documentation presents :</p>\n\n<h2 id=\"documents\">documents</h2>\n\n<ul>\n<li><a href=\"https://loco-philippe.github.io/ES/JSON%20semantic%20format%20(JSON-NTV).htm\">JSON-NTV specification</a> (Json for NTV data)</li>\n<li><a href=\"https://loco-philippe.github.io/ES/NTV%20tabular%20format%20(NTV-TAB).htm\">NTV-TAB specification</a> (Json for tabular data)</li>\n</ul>\n\n<h2 id=\"python-connectors-documentation\">Python Connectors documentation</h2>\n\n<ul>\n<li>API\n<ul>\n<li><a href=\"https://loco-philippe.github.io/ntv-pandas/ntv_pandas.html\">dev</a></li>\n<li><a href=\"https://loco-philippe.github.io/ntv-pandas/v2.0.1/ntv_pandas.html\">v2.0.1</a></li>\n<li><a href=\"https://loco-philippe.github.io/ntv-pandas/v2.0.0/ntv_pandas.html\">v2.0.0</a></li>\n<li><a href=\"https://loco-philippe.github.io/ntv-pandas/v1.1.1/ntv_pandas.html\">v1.1.1</a></li>\n<li><a href=\"https://loco-philippe.github.io/ntv-pandas/v1.1.0/ntv_pandas.html\">v1.1.0</a></li>\n<li><a href=\"https://loco-philippe.github.io/ntv-pandas/v1.0.2/ntv_pandas.html\">v1.0.2</a></li>\n<li><a href=\"https://loco-philippe.github.io/ntv-pandas/v1.0.1/ntv_pandas.html\">v1.0.1</a></li>\n<li><a href=\"https://loco-philippe.github.io/ntv-pandas/v1.0.0/ntv_pandas.html\">v1.0.0</a></li>\n<li><a href=\"https://loco-philippe.github.io/ntv-pandas/v0.1.1/ntv_pandas.html\">v0.1.1</a></li>\n</ul></li>\n<li>Release\n<ul>\n<li><a href=\"https://github.com/loco-philippe/ntv-pandas/tree/main/docs/release.rst\">all versions</a></li>\n</ul></li>\n</ul>\n\n<h1 id=\"roadmap\">Roadmap</h1>\n\n<ul>\n<li><strong>type extension</strong> : interval dtype and sparse format not yet included</li>\n<li><strong>table schema</strong> : add type / format (<code>geojson</code>/<code>topojson</code>, <code>geopoint</code>/<code>default</code>, <code>geopoint</code>/<code>object</code>, <code>duration</code>/<code>default</code>, <code>string</code>/<code>binary</code>, <code>string</code>/<code>uuid</code>),</li>\n<li><strong>null JSON data</strong> : strategy to define</li>\n<li><strong>multidimensional</strong> : merge tabular JSON format and multidimensional JSON format</li>\n<li><strong>pandas type</strong> : support for Series or DataFrame which include pandas data</li>\n<li><strong>data consistency</strong> : controls between NTVtype and NTVvalue</li>\n</ul>\n"}];
// mirrored in build-search-index.js (part 1)
// Also split on html tags. this is a cheap heuristic, but good enough.
elasticlunr.tokenizer.setSeperator(/[\s\-.;&_'"=,()]+|<[^>]*>/);
let searchIndex;
if (docs._isPrebuiltIndex) {
console.info("using precompiled search index");
searchIndex = elasticlunr.Index.load(docs);
} else {
console.time("building search index");
// mirrored in build-search-index.js (part 2)
searchIndex = elasticlunr(function () {
this.pipeline.remove(elasticlunr.stemmer);
this.pipeline.remove(elasticlunr.stopWordFilter);
this.addField("qualname");
this.addField("fullname");
this.addField("annotation");
this.addField("default_value");
this.addField("signature");
this.addField("bases");
this.addField("doc");
this.setRef("fullname");
});
for (let doc of docs) {
searchIndex.addDoc(doc);
}
console.timeEnd("building search index");
}
return (term) => searchIndex.search(term, {
fields: {
qualname: {boost: 4},
fullname: {boost: 2},
annotation: {boost: 2},
default_value: {boost: 2},
signature: {boost: 2},
bases: {boost: 2},
doc: {boost: 1},
},
expand: true
});
})();