deploy: 7080993

Author: fjxmlzn

_modules/pe/embedding/embedding.html

@@ -73,6 +73,7 @@ <h1>Source code for pe.embedding.embedding</h1><div class="highlight"><pre>
 <span></span><span class="kn">from</span> <span class="nn">abc</span> <span class="kn">import</span> <span class="n">ABC</span><span class="p">,</span> <span class="n">abstractmethod</span>
 
 <span class="kn">from</span> <span class="nn">pe.constant.data</span> <span class="kn">import</span> <span class="n">EMBEDDING_COLUMN_NAME</span>
+<span class="kn">from</span> <span class="nn">pe.data</span> <span class="kn">import</span> <span class="n">Data</span>
 
 
 <div class="viewcode-block" id="Embedding"><a class="viewcode-back" href="../../../api/pe.embedding.html#pe.embedding.embedding.Embedding">[docs]</a><span class="k">class</span> <span class="nc">Embedding</span><span class="p">(</span><span class="n">ABC</span><span class="p">):</span>
@@ -90,7 +91,19 @@ <h1>Source code for pe.embedding.embedding</h1><div class="highlight"><pre>
 <span class="sd">        :param data: The data to compute the embedding</span>
 <span class="sd">        :type data: :py:class:`pe.data.data.Data`</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="o">...</span></div></div>
+        <span class="o">...</span></div>
+
+<div class="viewcode-block" id="Embedding.filter_uncomputed_rows"><a class="viewcode-back" href="../../../api/pe.embedding.html#pe.embedding.embedding.Embedding.filter_uncomputed_rows">[docs]</a>    <span class="k">def</span> <span class="nf">filter_uncomputed_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="n">data_frame</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">data_frame</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">column_name</span> <span class="ow">in</span> <span class="n">data_frame</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
+            <span class="n">data_frame</span> <span class="o">=</span> <span class="n">data_frame</span><span class="p">[</span><span class="n">data_frame</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">column_name</span><span class="p">]</span><span class="o">.</span><span class="n">isna</span><span class="p">()]</span>
+        <span class="k">return</span> <span class="n">Data</span><span class="p">(</span><span class="n">data_frame</span><span class="o">=</span><span class="n">data_frame</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">metadata</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Embedding.merge_computed_rows"><a class="viewcode-back" href="../../../api/pe.embedding.html#pe.embedding.embedding.Embedding.merge_computed_rows">[docs]</a>    <span class="k">def</span> <span class="nf">merge_computed_rows</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">computed_data</span><span class="p">):</span>
+        <span class="n">data_frame</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">data_frame</span>
+        <span class="n">computed_data_frame</span> <span class="o">=</span> <span class="n">computed_data</span><span class="o">.</span><span class="n">data_frame</span>
+        <span class="n">data_frame</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">computed_data_frame</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">Data</span><span class="p">(</span><span class="n">data_frame</span><span class="o">=</span><span class="n">data_frame</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">metadata</span><span class="p">)</span></div></div>
 </pre></div>
 
            </div>

_modules/pe/embedding/image/inception.html

@@ -126,11 +126,15 @@ <h1>Source code for pe.embedding.image.inception</h1><div class="highlight"><pre
 <span class="sd">        :return: The data object with the computed embedding</span>
 <span class="sd">        :rtype: :py:class:`pe.data.data.Data`</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">column_name</span> <span class="ow">in</span> <span class="n">data</span><span class="o">.</span><span class="n">data_frame</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
+        <span class="n">uncomputed_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filter_uncomputed_rows</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">uncomputed_data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
             <span class="n">execution_logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Embedding: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">column_name</span><span class="si">}</span><span class="s2"> already computed&quot;</span><span class="p">)</span>
             <span class="k">return</span> <span class="n">data</span>
-        <span class="n">execution_logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Embedding: computing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">column_name</span><span class="si">}</span><span class="s2"> for </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">)</span><span class="si">}</span><span class="s2"> samples&quot;</span><span class="p">)</span>
-        <span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">stack</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">[</span><span class="n">IMAGE_DATA_COLUMN_NAME</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+        <span class="n">execution_logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="sa">f</span><span class="s2">&quot;Embedding: computing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">column_name</span><span class="si">}</span><span class="s2"> for </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">uncomputed_data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">)</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
+            <span class="s2">&quot; samples&quot;</span>
+        <span class="p">)</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">stack</span><span class="p">(</span><span class="n">uncomputed_data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">[</span><span class="n">IMAGE_DATA_COLUMN_NAME</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
         <span class="k">if</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
             <span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
         <span class="n">embeddings</span> <span class="o">=</span> <span class="p">[]</span>
@@ -146,9 +150,14 @@ <h1>Source code for pe.embedding.image.inception</h1><div class="highlight"><pre
             <span class="n">embeddings</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_inception</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">transformed_x</span><span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_device</span><span class="p">)))</span>
         <span class="n">embeddings</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">(</span><span class="n">embeddings</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
         <span class="n">embeddings</span> <span class="o">=</span> <span class="n">embeddings</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span>
-        <span class="n">data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">column_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">embeddings</span><span class="p">),</span> <span class="n">index</span><span class="o">=</span><span class="n">data</span><span class="o">.</span><span class="n">data_frame</span><span class="o">.</span><span class="n">index</span><span class="p">)</span>
-        <span class="n">execution_logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Embedding: finished computing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">column_name</span><span class="si">}</span><span class="s2"> for </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">)</span><span class="si">}</span><span class="s2"> samples&quot;</span><span class="p">)</span>
-        <span class="k">return</span> <span class="n">data</span></div></div>
+        <span class="n">uncomputed_data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">column_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">(</span>
+            <span class="nb">list</span><span class="p">(</span><span class="n">embeddings</span><span class="p">),</span> <span class="n">index</span><span class="o">=</span><span class="n">uncomputed_data</span><span class="o">.</span><span class="n">data_frame</span><span class="o">.</span><span class="n">index</span>
+        <span class="p">)</span>
+        <span class="n">execution_logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+            <span class="sa">f</span><span class="s2">&quot;Embedding: finished computing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">column_name</span><span class="si">}</span><span class="s2"> for &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">uncomputed_data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">)</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">data_frame</span><span class="p">)</span><span class="si">}</span><span class="s2"> samples&quot;</span>
+        <span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">merge_computed_rows</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">uncomputed_data</span><span class="p">)</span></div></div>
 </pre></div>
 
            </div>

_modules/pe/runner/pe.html

@@ -71,6 +71,7 @@
 
   <h1>Source code for pe.runner.pe</h1><div class="highlight"><pre>
 <span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
+<span class="kn">import</span> <span class="nn">copy</span>
 
 <span class="kn">from</span> <span class="nn">pe.dp</span> <span class="kn">import</span> <span class="n">Gaussian</span>
 <span class="kn">from</span> <span class="nn">pe.data</span> <span class="kn">import</span> <span class="n">Data</span>
@@ -243,8 +244,8 @@ <h1>Source code for pe.runner.pe</h1><div class="highlight"><pre>
                     <span class="n">syn_data_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">syn_data</span><span class="p">)</span>
                 <span class="n">syn_data</span> <span class="o">=</span> <span class="n">Data</span><span class="o">.</span><span class="n">concat</span><span class="p">(</span><span class="n">syn_data_list</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_priv_data</span><span class="o">.</span><span class="n">metadata</span><span class="p">)</span>
                 <span class="n">syn_data</span><span class="o">.</span><span class="n">data_frame</span><span class="o">.</span><span class="n">reset_index</span><span class="p">(</span><span class="n">drop</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+                <span class="n">syn_data</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_priv_data</span><span class="o">.</span><span class="n">metadata</span><span class="p">)</span>
                 <span class="n">syn_data</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">iteration</span> <span class="o">=</span> <span class="mi">0</span>
-                <span class="n">syn_data</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">label_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_priv_data</span><span class="o">.</span><span class="n">metadata</span><span class="o">.</span><span class="n">label_info</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">_log_metrics</span><span class="p">(</span><span class="n">syn_data</span><span class="p">)</span>
 
             <span class="c1"># Run PE iterations.</span>

api/pe.embedding.html

@@ -149,6 +149,16 @@ <h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this
 </dl>
 </dd></dl>
 
+<dl class="py method">
+<dt class="sig sig-object py" id="pe.embedding.embedding.Embedding.filter_uncomputed_rows">
+<span class="sig-name descname"><span class="pre">filter_uncomputed_rows</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pe/embedding/embedding.html#Embedding.filter_uncomputed_rows"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pe.embedding.embedding.Embedding.filter_uncomputed_rows" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
+<dl class="py method">
+<dt class="sig sig-object py" id="pe.embedding.embedding.Embedding.merge_computed_rows">
+<span class="sig-name descname"><span class="pre">merge_computed_rows</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">computed_data</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/pe/embedding/embedding.html#Embedding.merge_computed_rows"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#pe.embedding.embedding.Embedding.merge_computed_rows" title="Permalink to this definition"></a></dt>
+<dd></dd></dl>
+
 </dd></dl>
 
 </section>

genindex.html

@@ -319,12 +319,14 @@ <h2 id="F">F</h2>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api/pe.data.html#pe.data.data.Data.filter_label_id">filter_label_id() (pe.data.data.Data method)</a>
 </li>
-      <li><a href="api/pe.metric_item.html#pe.metric_item.FloatListMetricItem">FloatListMetricItem (class in pe.metric_item)</a>
+      <li><a href="api/pe.embedding.html#pe.embedding.embedding.Embedding.filter_uncomputed_rows">filter_uncomputed_rows() (pe.embedding.embedding.Embedding method)</a>
 </li>
-      <li><a href="api/pe.metric_item.html#pe.metric_item.FloatMetricItem">FloatMetricItem (class in pe.metric_item)</a>
+      <li><a href="api/pe.metric_item.html#pe.metric_item.FloatListMetricItem">FloatListMetricItem (class in pe.metric_item)</a>
 </li>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="api/pe.metric_item.html#pe.metric_item.FloatMetricItem">FloatMetricItem (class in pe.metric_item)</a>
+</li>
       <li><a href="api/pe.api.image.html#pe.api.image.improved_diffusion_api.Sampler.forward">forward() (pe.api.image.improved_diffusion_api.Sampler method)</a>
 </li>
       <li><a href="api/pe.api.image.improved_diffusion_lib.html#pe.api.image.improved_diffusion_lib.unet.FP32UNetModel">FP32UNetModel (class in pe.api.image.improved_diffusion_lib.unet)</a>
@@ -438,6 +440,8 @@ <h2 id="M">M</h2>
       <li><a href="api/pe.logger.html#pe.logger.matplotlib_pdf.MatplotlibPDF">MatplotlibPDF (class in pe.logger.matplotlib_pdf)</a>
 </li>
       <li><a href="api/pe.data.html#pe.data.data.Data.merge">merge() (pe.data.data.Data method)</a>
+</li>
+      <li><a href="api/pe.embedding.html#pe.embedding.embedding.Embedding.merge_computed_rows">merge_computed_rows() (pe.embedding.embedding.Embedding method)</a>
 </li>
       <li><a href="api/pe.metric_item.html#pe.metric_item.metric_scope">metric_scope (class in pe.metric_item)</a>
 </li>