Merge pull request #4 from xmos/feature/fseek

Feature fseek

.gitignore

@@ -150,3 +150,6 @@ _deps
 
 # Host binary
 xscope_fileio/host/xscope_host_endpoint
+*.bin
+*.o
+examples/fileio_features_xc/.build

CHANGELOG.rst

@@ -0,0 +1,20 @@
+xscope_fileo
+============
+
+0.2.0
+-----
+
+  * FIXED: setup.py now properly builds host app
+  * CHANGED: config.xscope now needs to be copied to local firmware app
+
+0.1.0
+-----
+
+  * ADDED: ftell and fseek
+  * ADDED: XC and C based examples
+  * CHANGED: module_build_info friendly layout of embedded source
+  * CHANGED: API argument ordering so file handle comes last as per stdio
+
+0.0.1
+-----
+  * ADDED: Initial version with support for fread & frwite + python helper

README.md

@@ -7,13 +7,11 @@ Currently it supports:
 
   * Arbitrary number (32 currently) of **read or write** files (not read/write)
 
-  * **Sequential** (not random) access only
-
-  * “wb” and “rb” file access mode only
+  * “wb” or “rb” file access mode only
 
   * 6-8MBytes/s Device to Host speed
 
-  * 150-850kBytes/s Host to Device speed (Linux < 200kB, Mac > 800kB)
+  * Up to 1MBytes/s Host to Device speed (on tools 15.0.4)
 
 This compares to around 2kBytes/s for fileio over JTAG supported using `xrun --io`.
 
@@ -51,17 +49,28 @@ void xscope_io_init(chanend_t xscope_end);
 
 xscope_file_t xscope_open_file(char* filename, char* attributes);
 
-size_t xscope_fread(uint8_t *buffer, size_t n_bytes_to_read, xscope_file_t *xscope_io_handle);
+size_t xscope_fread(xscope_file_t *xscope_io_handle, uint8_t *buffer, size_t n_bytes_to_read);
+
+void xscope_fwrite(xscope_file_t *xscope_io_handle, uint8_t *buffer, size_t n_bytes_to_write);
+
+void xscope_fseek(xscope_file_t *xscope_io_handle, int offset, int whence);
 
-void xscope_fwrite(uint8_t *buffer, size_t n_bytes_to_write, xscope_file_t *xscope_io_handle);
+int xscope_ftell(xscope_file_t *xscope_file);  
 
-void xscope_close_files(void);
+void xscope_close_all_files(void);
 ```
 
+The device side application requires a multi-tile main since it uses the xscope_host_data(xscope_chan); service
+to communicate with the host, which requires this. See examples for XC and C applications for how to do this.
+
+You will also need a copy of `config.xscope` in your firmware directory. This
+enables xscope in the tools and sets up the xscope probes used by fileio for communicating with the host app. You 
+can find a copy in `xscope_fileio/config.xscope xscope_fileio/config.xscope.txt` which you should rename to `config.xscope`.
+
 Note currently missing from fileio api:
 
 ```
-fseek, ftell, fprintf,  fscanf
+fprintf,  fscanf
 ```
 
 ## System Architecture

examples/fileio_features_xc/Makefile

@@ -0,0 +1,29 @@
+# The TARGET variable determines what target system the application is
+# compiled for. It either refers to an XN file in the source directories
+# or a valid argument for the --target option when compiling
+TARGET = XCORE-AI-EXPLORER
+
+# The APP_NAME variable determines the name of the final .xe file. It should
+# not include the .xe postfix. If left blank the name will default to
+# the project name
+APP_NAME = 
+
+# The USED_MODULES variable lists other modules used by the application.
+USED_MODULES = xscope_fileio(>=0.1.0)
+
+# The flags passed to xcc when building the application
+# You can also set the following to override flags for a particular language:
+# XCC_XC_FLAGS, XCC_C_FLAGS, XCC_ASM_FLAGS, XCC_CPP_FLAGS
+# If the variable XCC_MAP_FLAGS is set it overrides the flags passed to
+# xcc for the final link (mapping) stage.
+XCC_FLAGS = -Os -g -report -DTEST_WAV_XSCOPE=1
+
+# The XCORE_ARM_PROJECT variable, if set to 1, configures this
+# project to create both xCORE and ARM binaries.
+XCORE_ARM_PROJECT = 0
+
+# The VERBOSE variable, if set to 1, enables verbose output from the make system.
+VERBOSE = 0
+
+XMOS_MAKE_PATH ?= ../..
+-include $(XMOS_MAKE_PATH)/xcommon/module_xcommon/build/Makefile.common

src_xcore/config.xscope → examples/fileio_features_xc/config.xscope

@@ -3,6 +3,8 @@
   <Probe name="read_bytes" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
   <Probe name="write_setup" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
   <Probe name="write_bytes" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
+  <Probe name="seek" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
+  <Probe name="tell" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
   <Probe name="host_quit" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
 </xSCOPEconfig>
 

examples/fileio_features_xc/run_example.py

@@ -0,0 +1,26 @@
+#We assume that the Xscope FileIO Python library has been installed via pip beforehand and is available to import. Please see readme for instuctions.
+import subprocess
+import xscope_fileio
+
+
+firmware_xe = "bin/fileio_features_xc.xe"
+adapter_id = "L4Ss6YfM"
+ref_text = b"Evolution is change in the heritable characteristics of biological populations over successive generations." + b"\x00";
+with open("ref.bin", "wb") as ref_file:
+    ref_file.write(ref_text)
+
+xscope_fileio.run_on_target(adapter_id, firmware_xe, use_xsim=True)
+
+with open("dut.bin", "rb") as dut_file:
+    dut_text = dut_file.read()
+
+with open("dut_mod.bin", "rb") as dut_mod_file:
+    dut_mod_text = dut_mod_file.read()
+
+ref_mod_text = ref_text[0:10] + b"IS" + ref_text[12:]
+
+assert dut_text == ref_text
+print(dut_mod_text, ref_mod_text)
+assert dut_mod_text == ref_mod_text
+
+print("PASS")

examples/fileio_features_xc/test.xc

@@ -0,0 +1,76 @@
+#include <platform.h>
+#include <stdio.h>
+#include <string.h>
+#include <xscope.h>
+extern "C"{
+#include "xscope_io_device.h"
+}
+#include <assert.h>
+
+void test(void){
+    const char ref_file_name[] = "ref.bin";
+    const char ref_array[] = "Evolution is change in the heritable characteristics of biological populations over successive generations.";
+
+    xscope_file_t read_xscope_file = xscope_open_file(ref_file_name, "rb");
+    char buffer[256] = {0};
+
+    //Load in full ref array from disk
+    size_t num_bytes = xscope_fread(&read_xscope_file, buffer, sizeof(ref_array));
+    printf("Full sentence (%u): %s\n", num_bytes, buffer);
+
+    //Seek relative to start for read
+    xscope_fseek(&read_xscope_file, 10, SEEK_SET);
+    printf("Should say 10: %d\n", xscope_ftell(&read_xscope_file));
+
+    memset(buffer, 0, sizeof(buffer));
+    xscope_fread(&read_xscope_file, buffer, 2);
+    printf("Should say 'is': %s\n", buffer);
+
+    //Seek relative to end for read
+    xscope_fseek(&read_xscope_file, -29, SEEK_END);
+    printf("Should say 79: %d\n", xscope_ftell(&read_xscope_file));
+    memset(buffer, 0, sizeof(buffer));
+    xscope_fread(&read_xscope_file, buffer, 4);
+    printf("Should say 'over': %s\n", buffer);
+
+    //Seek relative to current for read
+    xscope_fseek(&read_xscope_file, 1, SEEK_CUR);
+    printf("Should say 84: %d\n", xscope_ftell(&read_xscope_file));
+    memset(buffer, 0, sizeof(buffer));
+    xscope_fread(&read_xscope_file, buffer, 10);
+    printf("Should say 'successive': %s\n", buffer);
+
+    //Load in full ref again
+    xscope_fseek(&read_xscope_file, 0, SEEK_SET);
+    printf("Should say 0: %d\n", xscope_ftell(&read_xscope_file));
+    num_bytes = xscope_fread(&read_xscope_file, buffer, sizeof(ref_array));
+    printf("Full sentence (%u): %s\n", num_bytes, buffer);
+
+    //Copy it out to dut for comparing
+    const char dut_file_name[] = "dut.bin";
+    xscope_file_t write_xscope_file = xscope_open_file(dut_file_name, "wb");
+    xscope_fwrite(&write_xscope_file, buffer, sizeof(ref_array));
+
+    //Test fseek on write file
+    const char dut_mod_file_name[] = "dut_mod.bin";
+    xscope_file_t write_xscope_fil_mod = xscope_open_file(dut_mod_file_name, "wb");
+    xscope_fwrite(&write_xscope_fil_mod, buffer, sizeof(ref_array));
+    xscope_fseek(&write_xscope_fil_mod, 10, SEEK_SET);  
+    printf("Should say 10: %d\n", xscope_ftell(&write_xscope_fil_mod));
+    xscope_fwrite(&write_xscope_fil_mod, "IS", 2);
+    printf("Should say 12: %d\n", xscope_ftell(&write_xscope_fil_mod));
+
+    xscope_close_all_files();
+}
+
+int main(void){
+    chan xscope_chan;
+    par{
+        xscope_host_data(xscope_chan);
+        on tile[0]:{
+                xscope_io_init(xscope_chan);
+                test();
+        }
+    }
+    return 0;
+}

examples/throughput_c/Makefile

@@ -0,0 +1,47 @@
+TARGET_FLAG ?= -target=XCORE-AI-EXPLORER
+TARGET_EXEC ?= fileio_test.xe
+
+XCC ?= xcc
+
+SRCS := main.xc test.c ../../xscope_fileio/src/xscope_io_device.c
+INC_DIRS += $(dir $(SRCS))
+INC_FLAGS := $(addprefix -I,$(INC_DIRS)) -I../../xscope_fileio/api -I../../xscope_fileio
+
+BUILD_DIR := build
+OBJS := $(SRCS:%=$(BUILD_DIR)/%.o)
+DEPS := $(OBJS:.o=.d)
+
+CFLAGS := -c $(INC_FLAGS) $(TARGET_FLAG) -Os -g
+XCFLAGS := $(CFLAGS)
+
+$(TARGET_EXEC): $(OBJS) config.xscope
+# 	@echo "**HELLO LINKER**"
+	$(XCC) $(TARGET_FLAG) -o $@ $^ -report
+
+# c source
+$(BUILD_DIR)/%.c.o: %.c
+	$(MKDIR_P) $(dir $@)
+# 	@echo "**HELLO C**"
+	$(XCC) $(CFLAGS) -c $< -o $@
+
+# xc source
+$(BUILD_DIR)/%.xc.o: %.xc
+	$(MKDIR_P) $(dir $@)
+# 	@echo "**HELLO XC**"
+	$(XCC) $(XCFLAGS) -c $< -o $@
+
+# asm source
+$(BUILD_DIR)/%.S.o: %.S
+	$(MKDIR_P) $(dir $@)
+# 	@echo "**HELLO ASM**"
+	$(XCC) $(ASMFLAGS) -c $< -o $@
+
+.PHONY: clean
+
+clean:
+	$(RM) -r build
+	$(RM) $(TARGET_EXEC)
+
+-include $(DEPS)
+
+MKDIR_P ?= mkdir -p

examples/throughput_c/config.xscope

@@ -0,0 +1,10 @@
+<xSCOPEconfig ioMode="basic" enabled="true">
+  <Probe name="open_file" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
+  <Probe name="read_bytes" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
+  <Probe name="write_setup" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
+  <Probe name="write_bytes" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
+  <Probe name="seek" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
+  <Probe name="tell" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
+  <Probe name="host_quit" type="CONTINUOUS" datatype="UINT" units="Value" enabled="true"/>
+</xSCOPEconfig>
+

examples/throughput_c/main.xc

@@ -0,0 +1,19 @@
+#include <platform.h>
+#include <xs1.h>
+#include <xscope.h>
+
+extern "C" {
+void main_tile0(chanend);
+}
+
+int main (void)
+{
+  chan xscope_chan;
+  par
+  {
+    xscope_host_data(xscope_chan);
+    on tile[0]: main_tile0(xscope_chan);
+  }
+  return 0;
+}
+

examples/throughput_c/run_example.py

@@ -0,0 +1,17 @@
+#We assume that the Xscope FileIO Python library has been installed via pip beforehand and is available to import. Please see readme for instuctions.
+import numpy as np
+import xscope_fileio
+
+firmware_xe = "fileio_test.xe"
+adapter_id = "L4Ss6YfM"
+
+ref = np.random.randint(256, size=(30 * 1024 * 1024)).astype(np.uint8)
+ref.tofile("ref.bin")
+
+xscope_fileio.run_on_target(adapter_id, firmware_xe, use_xsim=False)
+
+dut = np.fromfile("dut.bin", dtype=np.uint8)
+
+assert np.array_equal(ref, dut)
+
+print("PASS")

examples/throughput_c/test.c

@@ -0,0 +1,63 @@
+#include <platform.h>
+#include <stdio.h>
+#include <string.h>
+#include <xscope.h>
+#include <xcore/assert.h>
+#include <xcore/hwtimer.h>
+#include "xscope_io_device.h"
+
+#include <assert.h>
+
+#define IN_FILE_NAME    "ref.bin"
+#define OUT_FILE_NAME   "dut.bin"
+
+float ticks_to_KBPS(unsigned ticks, unsigned num_bytes){
+    const float ticks_per_second = 100000000;
+    float kb = (float)num_bytes / 1000;
+    float time_s = (float) ticks / ticks_per_second;
+
+    return kb/time_s;
+}
+
+void do_test(void){
+    xscope_file_t read_xscope_file = xscope_open_file(IN_FILE_NAME, "rb");
+    xscope_file_t write_xscope_file = xscope_open_file(OUT_FILE_NAME, "wb");
+
+    uint8_t buffer[256*1024] = {0};
+
+    xscope_fseek(&read_xscope_file, 0, SEEK_END);
+    unsigned fsize = xscope_ftell(&read_xscope_file);
+    xscope_fseek(&read_xscope_file, 0, SEEK_SET);
+    printf("Input file size kB: %d\n", fsize / 1000);
+
+    unsigned read_total_time = 0;
+    unsigned write_total_time = 0;
+
+
+    //Load in full ref array from disk
+    size_t num_bytes = 0;
+    do{
+        unsigned t0 = get_reference_time();
+        num_bytes = xscope_fread(&read_xscope_file, buffer, sizeof(buffer));
+        unsigned t1 = get_reference_time();
+        read_total_time += t1 - t0;
+
+        unsigned t2 = get_reference_time();
+        xscope_fwrite(&write_xscope_file, buffer, num_bytes);
+        unsigned t3 = get_reference_time();
+        write_total_time += t3 - t2;
+    } while(num_bytes > 0);
+
+
+
+
+
+    printf("Throughput KBPS Read: %f, Write: %f\n", ticks_to_KBPS(read_total_time, fsize), ticks_to_KBPS(write_total_time, fsize));
+}
+
+void main_tile0(chanend_t xscope_chan)
+{
+    xscope_io_init(xscope_chan);
+    do_test();
+    xscope_close_all_files();
+}

xscope_fileio/host/Makefile → host/Makefile

@@ -4,7 +4,7 @@ default_: all
 GCC ?= gcc
 
 xscope_host_endpoint : xscope_io_host.c
-	$(GCC) -o $@ $< -I ../../src_xcore -I ${XMOS_TOOL_PATH}/include ${XMOS_TOOL_PATH}/lib/xscope_endpoint.so -std=c99
+	$(GCC) -o $@ $< -I ../xscope_fileio -I ${XMOS_TOOL_PATH}/include ${XMOS_TOOL_PATH}/lib/xscope_endpoint.so -std=c99
 
 .PHONY: all
 all: xscope_host_endpoint