Publishing numpy string array from RTDPublisher corrupts heap

[andyvandy]

Hi Steven, hope the new year is treating you well!

I've tracked down a bug which is resulting in excel crashing suddenly with no warning. I believe it's tied to publishing numpy string arrays and when memory is freed on the python side.

Repro

Here's an example that can be run with a sheet locally. I've reproduced the crash on win10/win11, 32/64 bit, xloil 0.18.5/0.19.1 . I can provide more example cases if that would be helpful.

One additional case I need to send myself from another PC was if a global variable rather than local is published then the sheet doesn't crash. ( presumably since memory isn't freed )

import time
import xloil as xlo
import asyncio

TOPIC = "BUGHUNT"
_rtdServer = xlo.RtdServer()

async def crashes_1():
    _rtdServer.publish(TOPIC, [["danger"]])  
    await asyncio.sleep(10)
    _rtdServer.publish(TOPIC, [["safe!",1]])  

async def safe_1():
    _rtdServer.publish(TOPIC, [["no danger", 1]])  
    await asyncio.sleep(10)
    _rtdServer.publish(TOPIC, [["safe!",1]])  

async def safe_2():
    _rtdServer.publish(TOPIC, [["danger"]])  
    _rtdServer.publish(TOPIC, [["no danger", 1]])  
    await asyncio.sleep(10)
    _rtdServer.publish(TOPIC, [["safe!",1]])  

async def crashes_2():
    _rtdServer.publish(TOPIC, ["danger"])  

async def safe_3():
    _rtdServer.publish(TOPIC, "safe")  

async def crashes_3():
    _rtdServer.publish(TOPIC, ["danger"])  
    time.sleep(10)

async def safe_4():
    _rtdServer.publish(TOPIC, [["safe"],[1]])  

async def crashes_4():
    """Tuples aren't safe either..."""
    _rtdServer.publish(TOPIC, ("danger",))  

async def safe_6():
    """unless they aren't 1x1"""
    _rtdServer.publish(TOPIC, ("safe","safe"))  

TARGET_FUNC = crashes_4

class ExamplePublisher(xlo.RtdPublisher):
    def __init__(self):
        super().__init__()  # You *must* call this explicitly or the python binding library will crash
        self._task = None

    def connect(self, num_subscribers):
        if self.done():
            self._task = xlo.get_async_loop().create_task(TARGET_FUNC())

    def disconnect(self, num_subscribers):
        if num_subscribers == 0:
            self.real_stop()
            return True # This publisher is no longer required: schedule it for destruction

    def stop(self):
        pass

    def real_stop(self):
        if self._task is not None:
            self._task.cancel()

    def done(self):
        return self._task is None or self._task.done()

    def topic(self):
        return TOPIC

@xlo.func(local=False,rtd = True)  

def xloilRTDFunc( ):
    if _rtdServer.peek(TOPIC) is None:
        publisher = ExamplePublisher()
        _rtdServer.start(publisher)

    return _rtdServer.subscribe(TOPIC)

@xlo.func(local=False,rtd = True)  
async def xloilDirectRTDFunc( ):
    """Note that directly returning the values doesn't cause the same crashes"""
    yield "safe1"
    await asyncio.sleep(2)
    yield ["safe2"]
    await asyncio.sleep(2)
    yield [["safe3"]]
    await asyncio.sleep(2)
    yield ["safe4",1]
 

I managed to compile xloil locally and drop breakpoints. Unfortunately it has been hard to spot exactly where the issue is here since the crash doesn't occur right away. Please let me know if there's anything I can test or check which would be helpful in narrowing this down!