diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b32b382 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: CI + +on: [push] + +jobs: + lint: + name: Codestyle + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Stylua + uses: JohnnyMorganz/stylua-action@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + version: latest + args: --check . + docs: + name: Docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install pandoc + uses: pandoc/actions/setup@v1 + with: + version: 3.5 + - name: Regenerate docs + shell: bash + run: | + ./.panvimdoc/panvimdoc.sh + - name: Check that docs are up-to-date + run: git diff --exit-code -- doc/jupytext.txt + test: + name: Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: rhysd/action-setup-vim@v1 + id: vim + with: + neovim: true + version: v0.10.2 + - name: Install Jupytext + run: | + python -m pip install --upgrade pip + pip install jupytext + - name: Run test + shell: bash + run: | + ./run_tests.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..43bb4ba --- /dev/null +++ b/.gitignore @@ -0,0 +1,44 @@ +# Compiled Lua sources +luac.out + +# luarocks build files +*.src.rock +*.zip +*.tar.gz + +# Object files +*.o +*.os +*.ko +*.obj +*.elf + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo +*.def +*.exp + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +build/ +doc/tags +/.testenv/ diff --git a/.panvimdoc/panvimdoc.sh b/.panvimdoc/panvimdoc.sh new file mode 100755 index 0000000..6ed8680 --- /dev/null +++ b/.panvimdoc/panvimdoc.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Paths are from the perspective of the Makefile +PROJECT_NAME="jupytext" +INPUT_FILE="README.md" +DESCRIPTION="Edit .ipynb files" +SCRIPTS_DIR=".panvimdoc/scripts" +DOC_MAPPING=true + + +# Define arguments in an array +ARGS=( + "--shift-heading-level-by=${SHIFT_HEADING_LEVEL_BY:-0}" + "--metadata=project:$PROJECT_NAME" + "--metadata=vimversion:${VIM_VERSION:-""}" + "--metadata=toc:${TOC:-true}" + "--metadata=description:${DESCRIPTION:-""}" + "--metadata=titledatepattern:${TITLE_DATE_PATTERN:-"%Y %B %d"}" + "--metadata=dedupsubheadings:${DEDUP_SUBHEADINGS:-true}" + "--metadata=ignorerawblocks:${IGNORE_RAWBLOCKS:-true}" + "--metadata=docmapping:${DOC_MAPPING:-false}" + "--metadata=docmappingproject:${DOC_MAPPING_PROJECT_NAME:-true}" + "--metadata=treesitter:${TREESITTER:-true}" + "--metadata=incrementheadinglevelby:${INCREMENT_HEADING_LEVEL_BY:-0}" + "--lua-filter=$SCRIPTS_DIR/include-files.lua" + "--lua-filter=$SCRIPTS_DIR/skip-blocks.lua" +) + +ARGS+=("-t" "$SCRIPTS_DIR/panvimdoc.lua") + +# Print and execute the command +printf "%s\n" "pandoc --citeproc ${ARGS[*]} $INPUT_FILE -o doc/$PROJECT_NAME.txt" +pandoc "${ARGS[@]}" "$INPUT_FILE" -o "doc/$PROJECT_NAME.txt" diff --git a/.panvimdoc/scripts/include-files.lua b/.panvimdoc/scripts/include-files.lua new file mode 100644 index 0000000..eaee9ba --- /dev/null +++ b/.panvimdoc/scripts/include-files.lua @@ -0,0 +1,108 @@ +--- include-files.lua – filter to include Markdown files +--- +--- Copyright: © 2019–2021 Albert Krewinkel +--- License: MIT – see LICENSE file for details +-- Module pandoc.path is required and was added in version 2.12 +PANDOC_VERSION:must_be_at_least("3.0") + +local List = require("pandoc.List") +local path = require("pandoc.path") +local system = require("pandoc.system") + +function P(s) + require("scripts.logging").temp(s) +end + +--- Get include auto mode +local include_auto = false +function get_vars(meta) + if meta["include-auto"] then + include_auto = true + end +end + +--- Keep last heading level found +local last_heading_level = 0 +function update_last_level(header) + last_heading_level = header.level +end + +--- Update contents of included file +local function update_contents(blocks, shift_by, include_path) + local update_contents_filter = { + -- Shift headings in block list by given number + Header = function(header) + if shift_by then + header.level = header.level + shift_by + end + return header + end, + -- If image paths are relative then prepend include file path + Image = function(image) + if path.is_relative(image.src) then + image.src = path.normalize(path.join({ include_path, image.src })) + end + return image + end, + -- Update path for include-code-files.lua filter style CodeBlocks + CodeBlock = function(cb) + if cb.attributes.include and path.is_relative(cb.attributes.include) then + cb.attributes.include = path.normalize(path.join({ include_path, cb.attributes.include })) + end + return cb + end, + } + return pandoc.walk_block(pandoc.Div(blocks), update_contents_filter).content +end + +--- Filter function for code blocks +local transclude +function transclude(cb) + -- ignore code blocks which are not of class "include". + if not cb.classes:includes("include") then + return + end + + -- Markdown is used if this is nil. + local format = cb.attributes["format"] + + -- Attributes shift headings + local shift_heading_level_by = 0 + local shift_input = cb.attributes["shift-heading-level-by"] + if shift_input then + shift_heading_level_by = tonumber(shift_input) + else + if include_auto then + -- Auto shift headings + shift_heading_level_by = last_heading_level + end + end + + --- keep track of level before recusion + local buffer_last_heading_level = last_heading_level + + local blocks = List:new() + for line in cb.text:gmatch("[^\n]+") do + if line:sub(1, 2) ~= "//" then + -- local fh = io.open(pandoc.system.get_working_directory() .. line) + local mt, includecontents = pandoc.mediabag.fetch(line) + if not includecontents then + io.stderr:write("Cannot open file " .. line .. " | Skipping includes\n") + else + local contents = pandoc.read(includecontents, format).blocks + last_heading_level = 0 + -- recursive transclusion + contents = system.with_working_directory(path.directory(line), function() + return pandoc.walk_block(pandoc.Div(contents), { Header = update_last_level, CodeBlock = transclude }) + end).content + --- reset to level before recursion + last_heading_level = buffer_last_heading_level + blocks:extend(update_contents(contents, shift_heading_level_by, path.directory(line))) + -- fh:close() + end + end + end + return blocks +end + +return { { Meta = get_vars }, { Header = update_last_level, CodeBlock = transclude } } diff --git a/.panvimdoc/scripts/logging.lua b/.panvimdoc/scripts/logging.lua new file mode 100644 index 0000000..d2664bc --- /dev/null +++ b/.panvimdoc/scripts/logging.lua @@ -0,0 +1,263 @@ +--[[ + logging.lua: pandoc-aware logging functions (can also be used standalone) + Copyright: (c) 2022 William Lupton + License: MIT - see LICENSE file for details + Usage: See README.md for details +]] + +-- if running standalone, create a 'pandoc' global +if not pandoc then + _G.pandoc = { utils = {} } +end + +-- if there's no pandoc.utils, create a local one +if not pcall(require, "pandoc.utils") then + pandoc.utils = {} +end + +-- if there's no pandoc.utils.type, create a local one +if not pandoc.utils.type then + pandoc.utils.type = function(value) + local typ = type(value) + if not ({ table = 1, userdata = 1 })[typ] then + -- unchanged + elseif value.__name then + typ = value.__name + elseif value.tag and value.t then + typ = value.tag + if typ:match("^Meta.") then + typ = typ:sub(5) + end + if typ == "Map" then + typ = "table" + end + end + return typ + end +end + +-- namespace +local logging = {} + +-- helper function to return a sensible typename +logging.type = function(value) + -- this can return 'Inlines', 'Blocks', 'Inline', 'Block' etc., or + -- anything that built-in type() can return, namely 'nil', 'number', + -- 'string', 'boolean', 'table', 'function', 'thread', or 'userdata' + local typ = pandoc.utils.type(value) + + -- it seems that it can also return strings like 'pandoc Row'; replace + -- spaces with periods + -- XXX I'm not sure that this is done consistently, e.g. I don't think + -- it's done for pandoc.Attr or pandoc.List? + typ = typ:gsub(" ", ".") + + -- map Inline and Block to the tag name + -- XXX I guess it's intentional that it doesn't already do this? + return ({ Inline = 1, Block = 1 })[typ] and value.tag or typ +end + +-- derived from https://www.lua.org/pil/19.3.html pairsByKeys() +logging.spairs = function(list, comp) + local keys = {} + for key, _ in pairs(list) do + table.insert(keys, tostring(key)) + end + table.sort(keys, comp) + local i = 0 + local iter = function() + i = i + 1 + return keys[i] and keys[i], list[keys[i]] or nil + end + return iter +end + +-- helper function to dump a value with a prefix (recursive) +-- XXX should detect repetition/recursion +-- XXX would like maxlen logic to apply at all levels? but not trivial +local function dump_(prefix, value, maxlen, level, add) + local buffer = {} + if prefix == nil then + prefix = "" + end + if level == nil then + level = 0 + end + if add == nil then + add = function(item) + table.insert(buffer, item) + end + end + local indent = maxlen and "" or (" "):rep(level) + + -- get typename, mapping to pandoc tag names where possible + local typename = logging.type(value) + + -- don't explicitly indicate 'obvious' typenames + local typ = (({ boolean = 1, number = 1, string = 1, table = 1, userdata = 1 })[typename] and "" or typename) + + -- light userdata is just a pointer (can't iterate over it) + -- XXX is there a better way of checking for light userdata? + if type(value) == "userdata" and not pcall(pairs(value)) then + value = tostring(value):gsub("userdata:%s*", "") + + -- modify the value heuristically + elseif ({ table = 1, userdata = 1 })[type(value)] then + local valueCopy, numKeys, lastKey = {}, 0, nil + for key, val in pairs(value) do + -- pandoc >= 2.15 includes 'tag', nil values and functions + if key ~= "tag" and val and type(val) ~= "function" then + valueCopy[key] = val + numKeys = numKeys + 1 + lastKey = key + end + end + if numKeys == 0 then + -- this allows empty tables to be formatted on a single line + value = typename == "Space" and "" or "{}" + elseif numKeys == 1 and lastKey == "text" then + -- this allows text-only types to be formatted on a single line + typ = typename + value = value[lastKey] + typename = "string" + else + value = valueCopy + end + end + + -- output the possibly-modified value + local presep = #prefix > 0 and " " or "" + local typsep = #typ > 0 and " " or "" + local valtyp = type(value) + if valtyp == "nil" then + add("nil") + elseif ({ boolean = 1, number = 1, string = 1 })[valtyp] then + typsep = #typ > 0 and valtyp == "string" and #value > 0 and " " or "" + -- don't use the %q format specifier; doesn't work with multi-bytes + local quo = typename == "string" and "\"" or "" + add(string.format("%s%s%s%s%s%s%s%s", indent, prefix, presep, typ, typsep, quo, value, quo)) + -- light userdata is just a pointer (can't iterate over it) + -- XXX is there a better way of checking for light userdata? + elseif valtyp == "userdata" and not pcall(pairs(value)) then + add(string.format("%s%s%s%s %s", indent, prefix, presep, typ, tostring(value):gsub("userdata:%s*", ""))) + elseif ({ table = 1, userdata = 1 })[valtyp] then + add(string.format("%s%s%s%s%s{", indent, prefix, presep, typ, typsep)) + -- Attr and Attr.attributes have both numeric and string keys, so + -- ignore the numeric ones + -- XXX this is no longer the case for pandoc >= 2.15, so could remove + -- the special case? + local first = true + if prefix ~= "attributes:" and typ ~= "Attr" then + for i, val in ipairs(value) do + local pre = maxlen and not first and ", " or "" + local text = dump_(string.format("%s[%s]", pre, i), val, maxlen, level + 1, add) + first = false + end + end + -- report keys in alphabetical order to ensure repeatability + for key, val in logging.spairs(value) do + -- pandoc >= 2.15 includes 'tag' + if not tonumber(key) and key ~= "tag" then + local pre = maxlen and not first and ", " or "" + local text = dump_(string.format("%s%s:", pre, key), val, maxlen, level + 1, add) + end + first = false + end + add(string.format("%s}", indent)) + end + return table.concat(buffer, maxlen and "" or "\n") +end + +logging.dump = function(value, maxlen) + if maxlen == nil then + maxlen = 70 + end + local text = dump_(nil, value, maxlen) + if #text > maxlen then + text = dump_(nil, value, nil) + end + return text +end + +logging.output = function(...) + local need_newline = false + for i, item in ipairs({ ... }) do + -- XXX space logic could be cleverer, e.g. no space after newline + local maybe_space = i > 1 and " " or "" + local text = ({ table = 1, userdata = 1 })[type(item)] and logging.dump(item) or tostring(item) + io.stderr:write(maybe_space, text) + need_newline = text:sub(-1) ~= "\n" + end + if need_newline then + io.stderr:write("\n") + end +end + +-- basic logging support (-1=errors, 0=warnings, 1=info, 2=debug, 3=debug2) +-- XXX should support string levels? +logging.loglevel = 0 + +-- set log level and return the previous level +logging.setloglevel = function(loglevel) + local oldlevel = logging.loglevel + logging.loglevel = loglevel + return oldlevel +end + +-- verbosity default is WARNING; --quiet -> ERROR and --verbose -> INFO +-- --trace sets TRACE or DEBUG (depending on --verbose) +if type(PANDOC_STATE) == "nil" then + -- use the default level +elseif PANDOC_STATE.trace then + logging.loglevel = PANDOC_STATE.verbosity == "INFO" and 3 or 2 +elseif PANDOC_STATE.verbosity == "INFO" then + logging.loglevel = 1 +elseif PANDOC_STATE.verbosity == "WARNING" then + logging.loglevel = 0 +elseif PANDOC_STATE.verbosity == "ERROR" then + logging.loglevel = -1 +end + +logging.error = function(...) + if logging.loglevel >= -1 then + logging.output("(E)", ...) + end +end + +logging.warning = function(...) + if logging.loglevel >= 0 then + logging.output("(W)", ...) + end +end + +logging.info = function(...) + if logging.loglevel >= 1 then + logging.output("(I)", ...) + end +end + +logging.debug = function(...) + if logging.loglevel >= 2 then + logging.output("(D)", ...) + end +end + +logging.debug2 = function(...) + if logging.loglevel >= 3 then + logging.warning("debug2() is deprecated; use trace()") + logging.output("(D2)", ...) + end +end + +logging.trace = function(...) + if logging.loglevel >= 3 then + logging.output("(T)", ...) + end +end + +-- for temporary unconditional debug output +logging.temp = function(...) + logging.output("(#)", ...) +end + +return logging diff --git a/.panvimdoc/scripts/panvimdoc.lua b/.panvimdoc/scripts/panvimdoc.lua new file mode 100644 index 0000000..c98199c --- /dev/null +++ b/.panvimdoc/scripts/panvimdoc.lua @@ -0,0 +1,625 @@ +PANDOC_VERSION:must_be_at_least("3.0") + +local pipe = pandoc.pipe +local stringify = (require("pandoc.utils")).stringify +local text = pandoc.text + +function P(s) + require("scripts.logging").temp(s) +end + +-- custom writer for pandoc + +local unpack = unpack or table.unpack +local format = string.format +local stringify = pandoc.utils.stringify +local layout = pandoc.layout +local to_roman = pandoc.utils.to_roman_numeral + +function string.starts_with(str, starts) + return str:sub(1, #starts) == starts +end + +function string.ends_with(str, ends) + return ends == "" or str:sub(-#ends) == ends +end + +-- Character escaping +local function escape(s, in_attribute) + return s +end + +local function indent(s, fl, ol) + local ret = {} + local i = 1 + for l in s:gmatch("[^\r\n]+") do + if i == 1 then + ret[i] = fl .. l + else + ret[i] = ol .. l + end + i = i + 1 + end + return table.concat(ret, "\n") +end + +Writer = pandoc.scaffolding.Writer + +local function inlines(ils) + local buff = {} + for i = 1, #ils do + local el = ils[i] + buff[#buff + 1] = Writer[pandoc.utils.type(el)][el.tag](el) + end + return table.concat(buff) +end + +local function blocks(bs, sep, opts) + local dbuff = {} + for i = 1, #bs do + local el = bs[i] + local tag = el.tag + if tag == "Plain" then + -- We only get reflowing if we upgrade all text to "paragraph" + tag = "Para" + end + local writer = Writer[pandoc.utils.type(el)][tag] + local success, content = pcall(writer, el, opts) + if not success then + content = writer(el) + end + dbuff[#dbuff + 1] = content + end + return table.concat(dbuff, sep) +end + +local PROJECT = "" +local TREESITTER = false +local TOC = false +local VIMVERSION = "0.9.0" +local DESCRIPTION = "" +local DEDUP_SUBHEADINGS = false +local IGNORE_RAWBLOCKS = true +local DOC_MAPPING = true +local DOC_MAPPING_PROJECT = true +local DATE = nil +local TITLE_DATE_PATTERN = "%Y %B %d" + +local CURRENT_HEADER = nil +local SOFTBREAK_TO_HARDBREAK = "space" + +local HEADER_COUNT = 1 +local toc = {} +local links = {} + +local function osExecute(cmd) + local fileHandle = assert(io.popen(cmd, "r")) + local commandOutput = assert(fileHandle:read("*a")) + local returnTable = { fileHandle:close() } + return commandOutput, returnTable[3] -- rc[3] contains returnCode +end + +local function renderTitle() + local t = {} + local function add(s) + table.insert(t, s) + end + local vim_doc_title = PROJECT .. ".txt" + local vim_doc_title_tag = "*" .. vim_doc_title .. "*" + local project_description = DESCRIPTION or "" + if not project_description or #project_description == 0 then + local vim_version = VIMVERSION + if vim_version == nil then + vim_version = osExecute("nvim --version"):gmatch("([^\n]*)\n?")() + if string.find(vim_version, "-dev") then + vim_version = string.gsub(vim_version, "(.*)-dev.*", "%1") + end + if vim_version == "" then + vim_version = osExecute("vim --version"):gmatch("([^\n]*)\n?")() + vim_version = string.gsub(vim_version, "(.*) %(.*%)", "%1") + end + if vim_version == "" then + vim_version = "vim" + end + elseif vim_version == "vim" then + vim_version = osExecute("vim --version"):gmatch("([^\n]*)\n?")() + end + + local date = DATE + if date == nil then + date = os.date(TITLE_DATE_PATTERN) + end + local m = "For " .. vim_version + local r = "Last change: " .. date + local n = math.max(0, 78 - #vim_doc_title_tag - #m - #r) + local s = string.rep(" ", math.floor(n / 2)) + project_description = s .. m .. s .. r + end + local padding_len = math.max(0, 78 - #vim_doc_title_tag - #project_description) + add(vim_doc_title_tag .. string.rep(" ", padding_len) .. project_description) + add("") + return table.concat(t, "\n") +end + +local function renderToc() + if TOC then + local t = {} + local function add(s) + table.insert(t, s) + end + add(string.rep("=", 78)) + local l = "Table of Contents" + local tag = "*" .. PROJECT .. "-" .. string.gsub(string.lower(l), "%s", "-") .. "*" + add(l .. string.rep(" ", 78 - #l - #tag) .. tag) + add("") + for _, elem in pairs(toc) do + local level, item, link = elem[1], elem[2], elem[3] + if level == 1 then + local padding = string.rep(" ", 78 - #item - #link) + add(item .. padding .. link) + elseif level == 2 then + local padding = string.rep(" ", 74 - #item - #link) + add(" - " .. item .. padding .. link) + end + end + add("") + return table.concat(t, "\n") + else + return "" + end +end + +local function renderNotes() + local t = {} + local function add(s) + table.insert(t, s) + end + if #links > 0 then + local left = HEADER_COUNT .. ". Links" + local right = "links" + local right_link = string.format("|%s-%s|", PROJECT, right) + right = string.format("*%s-%s*", PROJECT, right) + local padding = string.rep(" ", 78 - #left - #right) + table.insert(toc, { 1, left, right_link }) + add(string.rep("=", 78) .. "\n" .. string.format("%s%s%s", left, padding, right)) + add("") + for i, v in ipairs(links) do + add(i .. ". *" .. v.caption .. "*" .. ": " .. v.src) + end + end + return table.concat(t, "\n") .. "\n" +end + +local function renderFooter() + return [[ +vim:tw=78:ts=8:noet:ft=help:norl:]] +end + +Writer.Pandoc = function(doc, opts) + PROJECT = doc.meta.project + TREESITTER = doc.meta.treesitter + TOC = doc.meta.toc + VIMVERSION = doc.meta.vimversion + DESCRIPTION = doc.meta.description + DEDUP_SUBHEADINGS = doc.meta.dedupsubheadings + IGNORE_RAWBLOCKS = doc.meta.ignorerawblocks + DOC_MAPPING = doc.meta.docmapping + DOC_MAPPING_PROJECT = doc.meta.docmappingproject + HEADER_COUNT = HEADER_COUNT + doc.meta.incrementheadinglevelby + DATE = doc.meta.date + TITLE_DATE_PATTERN = doc.meta.titledatepattern + local d = blocks(doc.blocks) + local notes = renderNotes() + local toc = renderToc() + local title = renderTitle() + local footer = renderFooter() + return { title, layout.blankline, toc, d, notes, layout.blankline, footer } +end + +Writer.Block.Header = function(el) + local lev = el.level + local s = stringify(el) + local attr = el.attr + local left, right, right_link, padding + if lev == 1 then + left = string.format("%d. %s", HEADER_COUNT, s) + right = string.lower(string.gsub(s, "%s", "-")) + CURRENT_HEADER = right + right_link = string.format("|%s-%s|", PROJECT, right) + right = string.format("*%s-%s*", PROJECT, right) + padding = string.rep(" ", 78 - #left - #right) + table.insert(toc, { 1, left, right_link }) + s = string.format("%s%s%s", left, padding, right) + HEADER_COUNT = HEADER_COUNT + 1 + s = string.rep("=", 78) .. "\n" .. s + return "\n" .. s .. "\n\n" + end + if lev == 2 then + left = string.upper(s) + right = string.lower(string.gsub(s, "%s", "-")) + if DEDUP_SUBHEADINGS and CURRENT_HEADER then + right_link = string.format("|%s-%s-%s|", PROJECT, CURRENT_HEADER, right) + right = string.format("*%s-%s-%s*", PROJECT, CURRENT_HEADER, right) + else + right_link = string.format("|%s-%s|", PROJECT, right) + right = string.format("*%s-%s*", PROJECT, right) + end + padding = string.rep(" ", 76 - #left - #right) + table.insert(toc, { 2, s, right_link }) + s = string.format("%s%s%s", left, padding, right) + return "\n" .. s .. "\n\n" + end + if lev == 3 then + left = s + return "\n" .. left .. " ~" .. "\n\n" + end + if lev == 4 then + if DOC_MAPPING then + left = s + if el.content.tag == Code then + left = '`' .. s .. '`' + end + if attr.attributes.doc then + right = "*" .. attr.attributes.doc .. "*" + elseif DOC_MAPPING_PROJECT then + -- stylua: ignore + right = string.format( + "*%s-%s*", + PROJECT, + s:gsub("{.+}", "") + :gsub("%[.+%]", "") + :gsub("^%s*(.-)%s*$", "%1") + :gsub("^%s*(.-)%s*$", "%1") + :gsub("%s", "-") + ) + else + -- stylua: ignore + right = string.format( + "*%s*", + s:gsub("{.+}", "") + :gsub("%[.+%]", "") + :gsub("^%s*(.-)%s*$", "%1") + :gsub("^%s*(.-)%s*$", "%1") + :gsub("%s", "-") + ) + end + padding = string.rep(" ", 78 - #left - #right) + local r = string.format("%s%s%s", left, padding, right) + return "\n" .. r .. "\n\n" + else + left = string.upper(s) + return "\n" .. left .. "\n\n" + end + end + if lev >= 5 then + left = string.upper(s) + return "\n" .. left .. "\n\n" + end +end + +Writer.Block.Para = function(el, opts) + local s = inlines(el.content) + local t = {} + local current_line = "" + local line_length = 78 + if opts and opts.line_length then + line_length = opts.line_length + end + for word in string.gmatch(s, "([^%s]+)") do + if string.match(word, "[.]") and #word == 1 then + current_line = current_line .. word + elseif (#current_line + #word) >= line_length then + table.insert(t, current_line) + current_line = word + elseif #current_line == 0 then + current_line = word + else + current_line = current_line .. " " .. word + end + end + table.insert(t, current_line) + return table.concat(t, "\n") .. "\n\n" +end + +Writer.Block.OrderedList = function(items) + local buffer = {} + local i = 1 + items.content:map(function(item) + table.insert(buffer, ("%s. %s"):format(i, blocks(item))) + i = i + 1 + end) + return table.concat(buffer, "\n") .. "\n\n" +end + +Writer.Block.BulletList = function(items) + local buffer = {} + items.content:map(function(item) + table.insert(buffer, indent(blocks(item, "\n", {line_length=76}), "- ", " ")) + end) + return table.concat(buffer, "\n") .. "\n\n" +end + +Writer.Block.DefinitionList = function(el) + local buffer = {} + local function add(s) + table.insert(buffer, s) + end + el.content:map(function(item) + local k = inlines(item[1]) + local bs = item[2][1] + local t = {} + for i = 1, #bs do + local e = bs[i] + if e.tag == "Para" then + local tt = {} + e.content:map(function(i) + if i.tag == "SoftBreak" then + table.insert(tt, "\n") + else + table.insert(tt, Writer[pandoc.utils.type(i)][i.tag](i)) + end + end) + table.insert(t, table.concat(tt)) + else + table.insert(t, Writer[pandoc.utils.type(e)][e.tag](e)) + end + end + local str = table.concat(t, "\n") + local i = 1 + + local right = "" + if DOC_MAPPING_PROJECT then + -- stylua: ignore + right = string.format( + "*%s-%s*", + PROJECT, + k:gsub("{.+}", "") + :gsub("%[.+%]", "") + :gsub("^%s*(.-)%s*$", "%1") + :gsub("^%s*(.-)%s*$", "%1") + :gsub("%s", "-") + ) + else + -- stylua: ignore + right = string.format( + "*%s*", + k:gsub("{.+}", "") + :gsub("%[.+%]", "") + :gsub("^%s*(.-)%s*$", "%1") + :gsub("^%s*(.-)%s*$", "%1") + :gsub("%s", "-") + ) + end + add(string.rep(" ", 78 - #right - 2) .. right) + add("\n") + for s in str:gmatch("[^\r\n]+") do + if i == 1 then + add(k .. string.rep(" ", 78 - 40 + 1 - #k) .. s) + else + add(string.rep(" ", 78 - 40 + 1) .. s) + end + i = i + 1 + end + add("\n") + end) + return "\n" .. table.concat(buffer, "\n") .. "\n\n" +end + +Writer.Block.CodeBlock = function(el) + local attr = el.attr + local s = el.text + if #attr.classes > 0 and attr.classes[1] == "vimdoc" then + return s .. "\n\n" + else + local lang = "" + if TREESITTER and #attr.classes > 0 then + lang = attr.classes[1] + end + local t = {} + for line in s:gmatch("([^\n]*)\n?") do + table.insert(t, " " .. escape(line)) + end + return ">" .. lang .. "\n" .. table.concat(t, "\n") .. "\n<\n\n" + end +end + +Writer.Inline.Str = function(el) + local s = stringify(el) + if string.starts_with(s, "(http") and string.ends_with(s, ")") then + return " <" .. string.sub(s, 2, #s - 2) .. ">" + else + return escape(s) + end +end + +Writer.Inline.Space = function() + return " " +end + +Writer.Inline.SoftBreak = function() + if SOFTBREAK_TO_HARDBREAK == "newline" then + return "\n" + elseif SOFTBREAK_TO_HARDBREAK == "space" then + return "\n" + else + return "" + end +end + +Writer.Inline.LineBreak = function() + return "\n" +end + +Writer.Inline.Emph = function(s) + return "_" .. stringify(s) .. "_" +end + +Writer.Inline.Strong = function(s) + return "**" .. stringify(s) .. "**" +end + +Writer.Inline.Subscript = function(s) + return "_" .. stringify(s) +end + +Writer.Inline.Superscript = function(s) + return "^" .. stringify(s) +end + +Writer.Inline.SmallCaps = function(s) + return stringify(s) +end + +Writer.Inline.Strikeout = function(s) + return "~" .. stringify(s) .. "~" +end + +Writer.Inline.Link = function(el) + local s = inlines(el.content) + local tgt = el.target + local tit = el.title + local attr = el.attr + if string.starts_with(tgt, "https://neovim.io/doc/") then + return "|" .. s .. "|" + elseif string.starts_with(tgt, "#") then + return "|" .. PROJECT .. "-" .. s:lower():gsub("%s", "-") .. "|" + elseif string.starts_with(s, "http") then + return "<" .. s .. ">" + else + return s .. " <" .. tgt .. ">" + end +end + +Writer.Inline.Image = function(el) + links[#links + 1] = { caption = inlines(el.caption), src = el.src } +end + +Writer.Inline.Code = function(el) + local content = stringify(el) + local vim_help = string.match(content, "^:h %s*([^%s]+)") + if vim_help then + return string.format("|%s|", escape(vim_help)) + else + return "`" .. escape(content) .. "`" + end +end + +Writer.Inline.Math = function(s) + return "`" .. escape(stringify(s)) .. "`" +end + +Writer.Inline.Quoted = function(el) + if el.quotetype == "DoubleQuote" then + return "\"" .. inlines(el.content) .. "\"" + else + return "'" .. inlines(el.content) .. "'" + end +end + +Writer.Inline.Note = function(el) + return stringify(el) +end + +Writer.Inline.Null = function(s) + return "" +end + +Writer.Inline.Span = function(el) + return inlines(el.content) +end + +Writer.Inline.RawInline = function(el) + if IGNORE_RAWBLOCKS then + return "" + end + local str = el.text + if format == "html" then + if str == "" then + return "" + elseif str == "" then + return " ~" + elseif str == "" or str == "" then + return "_" + elseif str == "" or str == "" then + return "" + else + return str + end + else + return "" + end +end + +Writer.Inline.Citation = function(el) + return el +end + +Writer.Inline.Cite = function(el) + links[#links + 1] = { caption = inlines(el.content), src = "" } + return inlines(el.content) +end + +Writer.Block.Plain = function(el) + return inlines(el.content) +end + +Writer.Block.RawBlock = function(el) + local fmt = el.format + local str = el.text + if fmt == "html" then + if string.starts_with(str, "" then + COMMENT = true + return pandoc.List() + elseif str == "" then + COMMENT = false + return pandoc.List() + end + if + ( + string.starts_with(str, "") + then + local content = string.match(str, "") + return pandoc.read(content, "markdown").blocks + end + if string.starts_with(str, "" then + COMMENT = true + return pandoc.Str("") + elseif str == "" then + COMMENT = false + return pandoc.Str("") + end + if COMMENT == true then + return pandoc.Str("") + end + return el +end + +function SmallCaps(el) + if COMMENT == true then + return pandoc.Str("") + end + return el +end + +function SoftBreak(el) + if COMMENT == true then + return pandoc.Str("") + end + return el +end + +function Space(el) + if COMMENT == true then + return pandoc.Str("") + end + return el +end + +function Span(el) + if COMMENT == true then + return pandoc.Str("") + end + return el +end + +function Text(el) + if COMMENT == true then + return pandoc.Str("") + end + return el +end + +function Strikeout(el) + if COMMENT == true then + return pandoc.Str("") + end + return el +end + +function Strong(el) + if COMMENT == true then + return pandoc.Str("") + end + return el +end + +function Subscript(el) + if COMMENT == true then + return pandoc.Str("") + end + return el +end + +function Superscript(el) + if COMMENT == true then + return pandoc.Str("") + end + return el +end + +function Underline(el) + if COMMENT == true then + return pandoc.Str("") + end + return el +end diff --git a/.stylua.toml b/.stylua.toml new file mode 100644 index 0000000..a2b3447 --- /dev/null +++ b/.stylua.toml @@ -0,0 +1,6 @@ +column_width = 100 +line_endings = "Unix" +indent_type = "Spaces" +indent_width = 2 +quote_style = "AutoPreferSingle" +call_parentheses = "Always" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7c27830 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Michael Goerz + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6c5bfa4 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +.PHONY: help codestyle doc test clean + +.DEFAULT_GOAL := help + +help: ## Show this help + @grep -E '^([a-zA-Z_-]+):.*## ' $(MAKEFILE_LIST) | awk -F ':.*## ' '{printf "%-20s %s\n", $$1, $$2}' + + +codestyle: ## Apply code formatting + stylua lua/*.lua lua/jupytext/*.lua tests/*.lua + + +doc: doc/jupytext.txt ## Generate documentation from README + +test: ## Run the test suite + ./run_tests.sh + +clear: ## Remove generated files + rm -rf .testenv + +doc/jupytext.txt: README.md + ./.panvimdoc/panvimdoc.sh + diff --git a/README.md b/README.md new file mode 100644 index 0000000..285fac1 --- /dev/null +++ b/README.md @@ -0,0 +1,158 @@ +jupytext.nvim +============= + + + +[![CI](https://github.com/goerz/jupytext.nvim/actions/workflows/ci.yml/badge.svg)](https://github.com/goerz/jupytext.nvim/actions/workflows/ci.yml) + + + +The plugin enables editing [Jupyter notebook `.ipynb` files](https://jupyter.org) as plain text files by dynamically converting them through the [`jupytext` command line tool](https://github.com/mwouts/jupytext). + +It is a rewrite of the [`jupytext.vim` plugin](https://github.com/goerz/jupytext.vim) for Neovim. Compared to the initial Vimscript version of the plugin, it should behave more robustly and be more flexible. For example, it can auto-detect the language used in the notebook. Most importantly, though, it is designed to integrate well with setups that use [paired files](#paired-files). + + +Prerequisites +============= + +The [`jupytext` command line utility](https://github.com/mwouts/jupytext) must be installed. + +It is recommended to also have [`jq`](https://jqlang.github.io/jq/) installed. + +After following the [installation](#installation) instructions, the `:checkhealth jupytext` command can be used inside Neovim to verify the prerequisites. + + +Installation +============ + +Load the plugin via your favorite package manager. [`Lazy.nvim`](https://lazy.folke.io) is recommended. + +### Lazy.nvim + +Use the following plugin specification: + +```lua +{ + 'goerz/jupytext.nvim', + opts = {}, -- see Options +} +``` + +### Manual setup + +If you are not using a package manager, copy the `lua/jupytext.lua` file into your runtime folder (e.g., `~/.config/nvim/`). Then, in your `init.lua` file, call `require("jupytext").setup(opts)` where `opts` is a table that may contain any of the keys listed in [Options](#options). + + +Options +======= + +The default options are: + +```lua +opts = { + jupytext = 'jupytext', + format = "markdown", + update = true, + filetype = require("jupytext").get_filetype, + sync_patterns = { '*.md', '*.py', '*.jl', '*.R', '*.Rmd', '*.qmd' }, + autosync = true, +} +``` + +#### `jupytext` + +The `jupytext` command to use. If `jupytext` is not on your `PATH`, you could set an absolute path here. Can also be set via a `b:jupytext_jupytext` variable. + +#### `format` + +The plain text format to use. See the description of `OUTPUT_FORMAT` in `jupytext --help`: `'markdown'` or `'script'`, or a file extension: `'md'`, `'Rmd'`, `'jl'`, `'py'`, `'R'`, …, `'auto'` (script extension matching the notebook language), or a combination of an extension and a format name, e.g., `'md:myst'`, `'md:markdown'`, `'md:pandoc'`, or `'py:percent'`, `'py:light'`, `'py:sphinx'`, `'py:hydrogen'`. + +The `format` option may also be given as a function that calculates the `format` from two arguments: + +- `path`: the absolute path to the `.ipynb` file being loaded +- `metadata`: a table with metadata information from the original JSON data in the `.ipynb` file. Of particular interest may be the field `metadata.jupytext.formats` if one wanted to implement something where paired notebooks would preferentially use the paired format instead of a common default value. + +To set the `format` option temporarily on a per-buffer basis, set `b:jupytext_format` and reload the file. + +#### `update` + +Whether or not to use the `--update` flag to `jupytext`. If `true` (recommended), this preserves existing outputs in the edited `.ipynb` file. If `false`, every save clears all outputs from the underlying file. + +This can be temporarily overridden with a `b:jupytext_update` variable. + +#### `filetype` + +The buffer `filetype` setting to use after loading the file, which determines syntax highlighting, etc. Can be given as a string, or more commonly as a function that returns a string after receiving three arguments: + +- `path`: the absolute path to the `.ipynb` file being loaded +- `format`: the value of the `format` option +- `metadata`: a table with metadata information from the original JSON data in the `.ipynb` file. This should contain, e.g., the notebook language in `metadata.kernelspec.language` + +The default function used for this setting uses `"markdown"` for markdown formats, and the value of `metadata.kernelspec.language` otherwise. Like the previous options, `b:jupytext_filetype` is available to temporarily override the choice of filetype. + +#### `sync_pattern` + +Patterns for plain text files that should be recognized as "syncable". If `autosync=true` (see below), and if, for a file matching the patterns, there also exists a file with an `.ipynb` extension, autocommands will be set up for `jupyter --sync` to be called before loading the file and after saving the file. This also periodically calls the `:checktime` function in the background to determine whether the file has changed on disk (by a running Jupyter server, presumably), and reloads it when appropriate. + +Note that the `sync_pattern` only determines for which plain text files the appropriate autocommands will be set up in Neovim. The setting is independent of which Jupytext pairings are active, which is in the metadata for the `.ipynb` files. All linked files will automatically be kept in sync. Likewise, when editing `.ipynb` files directly, _all_ linked files will be kept in sync automatically if `autosync=true`, irrespective of `sync_pattern`. + + +#### `autosync` + +If true (recommended), enable automatic synchronization for files paired via the Jupytext plugin (the plugin for Jupyter Lab). For `.ipynb` files, this checks if the notebook is paired to any plain text files. If so, it will call `jupytext --sync` before loading the file and after saving it, to ensure all files are being kept in sync. It will also periodically check whether the `.ipynb` file has changed on disk and reload it if necessary, setting the [`autoread`](https://neovim.io/doc/user/options.html#'autoread') option for the current buffer. + +Editing paired files while `autosync = false` will unpair them. + + +Usage +===== + +When opening an `.ipynb` file, this plugin will inject itself into the loading process and convert the `json` data in the file to a plain text format by piping it through `jupytext` using the `format` set in [Options](#options). + +On saving, the original `.ipynb` file will be updated by piping the content of the current buffer back into `jupytext`. With the default `update` setting, this will keep existing outputs and metadata in the notebook. + + +Paired Files +============ + +While the Jupytext project provides the command line utility `jupytext` used by this plugin for on-the-fly conversion between `.ipynb` and plain text formats, its _primary_ purpose is to provide a plugin for Jupyter to _pair_ `.ipynb` files with one or more text files that are easier to manage in version control. + +The intent of the original `jupytext.vim` plugin was to edit `.ipynb` files _not_ paired in such a way, and not loaded in an active Jupyter session. With this rewritten version of the plugin, `jupytext.nvim` now supports editing `.ipynb` files with Neovim if they are paired in Jupyter, and, at least in principle, even while the notebooks are actively loaded in a running Jupyter server. + +For this to work, the `autosync` option must be set to `true` (default, see [Options](#options)). This automatically handles the update of any paired files and watches for modifications of the file on disk while it is being edited. + + + +Even though editing files that are also actively loaded in Jupyter _works_, it might still be preferable to close the file in Jupyter first. The support in Jupyter for detecting external changes is not quite as good. You will have to manually reload files after saving them in Neovim. In the future, it might be possible to [combine the Jupytext plugin for Jupyter with its real-time-collaboration plugin](https://github.com/jupyterlab/jupyter-collaboration/issues/214), which would alleviate this concern. + + +Development +=========== + +During development, `make` can be used locally to apply the Lua code style, generate the documentation, and run the tests. See `make help` for details. + +Pushing commits to GitHub verifies the code formatting, the tests, and that the documentation is up-to-date via GitHub Actions. + +### Documentation + +The documentation for the plugin is maintained in this `README` file. This must be kept in sync with the [vim help format](doc/jupytext.txt). Running `make doc` locally regenerates the Vim help file from the current `README` to ensure this. The conversion relies on [`pandoc`](https://pandoc.org) and a number of [custom filters](.panvimdoc/scripts/) adapted from [`kdheepak/panvimdoc`](https://github.com/kdheepak/panvimdoc). See [its documentation](https://raw.githubusercontent.com/kdheepak/panvimdoc/refs/heads/main/doc/panvimdoc.md) for details on the recommended markdown syntax to use. + +GitHub Actions will check that the `README` and the Vim help file are in sync. + + +### Testing + +This plugin uses the [`plenary.nvim` test framework](https://github.com/nvim-lua/plenary.nvim/blob/master/TESTS_README.md). Tests are organized in `tests/*_spec.lua` files, and can be run by executing the `./run_tests.sh` script. This also happens automatically on GitHub Actions with each push. + + + + +History +======= + +### v.0.1.0-dev (unreleased) + +- Initial release after complete rewrite from `jupytext.vim` +- Avoid the use of temporary files in the same folder as the `.ipynb` files, clashing with paired scripts +- Added support for obtaining the notebook language from its metadata, enabling use of the "script" format +- Added support for automatic synchronization with paired files. diff --git a/doc/jupytext.txt b/doc/jupytext.txt new file mode 100644 index 0000000..96749ea --- /dev/null +++ b/doc/jupytext.txt @@ -0,0 +1,219 @@ +*jupytext.txt* Edit .ipynb files + +============================================================================== +Table of Contents *jupytext-table-of-contents* + +1. jupytext.nvim |jupytext-jupytext.nvim| +2. Prerequisites |jupytext-prerequisites| +3. Installation |jupytext-installation| +4. Options |jupytext-options| +5. Usage |jupytext-usage| +6. Paired Files |jupytext-paired-files| +7. History |jupytext-history| + +============================================================================== +1. jupytext.nvim *jupytext-jupytext.nvim* + +The plugin enables editing Jupyter notebook `.ipynb` files + as plain text files by dynamically converting them +through the `jupytext` command line tool . + +It is a rewrite of the `jupytext.vim` plugin + for Neovim. Compared to the initial +Vimscript version of the plugin, it should behave more robustly and be more +flexible. For example, it can auto-detect the language used in the notebook. +Most importantly, though, it is designed to integrate well with setups that +use |jupytext-paired-files|. + + +============================================================================== +2. Prerequisites *jupytext-prerequisites* + +The `jupytext` command line utility must +be installed. + +It is recommended to also have `jq` installed. + +After following the |jupytext-installation| instructions, the `:checkhealth +jupytext` command can be used inside Neovim to verify the prerequisites. + + +============================================================================== +3. Installation *jupytext-installation* + +Load the plugin via your favorite package manager. `Lazy.nvim` + is recommended. + + +Lazy.nvim ~ + +Use the following plugin specification: + +>lua + { + 'goerz/jupytext.nvim', + opts = {}, -- see Options + } +< + + +Manual setup ~ + +If you are not using a package manager, copy the `lua/jupytext.lua` file into +your runtime folder (e.g., `~/.config/nvim/`). Then, in your `init.lua` file, +call `require("jupytext").setup(opts)` where `opts` is a table that may +contain any of the keys listed in |jupytext-options|. + + +============================================================================== +4. Options *jupytext-options* + +The default options are: + +>lua + opts = { + jupytext = 'jupytext', + format = "markdown", + update = true, + filetype = require("jupytext").get_filetype, + sync_patterns = { '*.md', '*.py', '*.jl', '*.R', '*.Rmd', '*.qmd' }, + autosync = true, + } +< + + +`jupytext` *jupytext-jupytext* + +The `jupytext` command to use. If `jupytext` is not on your `PATH`, you could +set an absolute path here. Can also be set via a `b:jupytext_jupytext` +variable. + + +`format` *jupytext-format* + +The plain text format to use. See the description of `OUTPUT_FORMAT` in +`jupytext --help`: `'markdown'` or `'script'`, or a file extension: `'md'`, +`'Rmd'`, `'jl'`, `'py'`, `'R'`, …, `'auto'` (script extension matching the +notebook language), or a combination of an extension and a format name, e.g., +`'md:myst'`, `'md:markdown'`, `'md:pandoc'`, or `'py:percent'`, `'py:light'`, +`'py:sphinx'`, `'py:hydrogen'`. + +The `format` option may also be given as a function that calculates the +`format` from two arguments: + +- `path`: the absolute path to the `.ipynb` file being loaded +- `metadata`: a table with metadata information from the original JSON data in + the `.ipynb` file. Of particular interest may be the field + `metadata.jupytext.formats` if one wanted to implement something where + paired notebooks would preferentially use the paired format instead of a + common default value. + +To set the `format` option temporarily on a per-buffer basis, set +`b:jupytext_format` and reload the file. + + +`update` *jupytext-update* + +Whether or not to use the `--update` flag to `jupytext`. If `true` +(recommended), this preserves existing outputs in the edited `.ipynb` file. If +`false`, every save clears all outputs from the underlying file. + +This can be temporarily overridden with a `b:jupytext_update` variable. + + +`filetype` *jupytext-filetype* + +The buffer `filetype` setting to use after loading the file, which determines +syntax highlighting, etc. Can be given as a string, or more commonly as a +function that returns a string after receiving three arguments: + +- `path`: the absolute path to the `.ipynb` file being loaded +- `format`: the value of the `format` option +- `metadata`: a table with metadata information from the original JSON data in + the `.ipynb` file. This should contain, e.g., the notebook language in + `metadata.kernelspec.language` + +The default function used for this setting uses `"markdown"` for markdown +formats, and the value of `metadata.kernelspec.language` otherwise. Like the +previous options, `b:jupytext_filetype` is available to temporarily override +the choice of filetype. + + +`sync_pattern` *jupytext-sync_pattern* + +Patterns for plain text files that should be recognized as "syncable". If +`autosync=true` (see below), and if, for a file matching the patterns, there +also exists a file with an `.ipynb` extension, autocommands will be set up for +`jupyter --sync` to be called before loading the file and after saving the +file. This also periodically calls the `:checktime` function in the background +to determine whether the file has changed on disk (by a running Jupyter +server, presumably), and reloads it when appropriate. + +Note that the `sync_pattern` only determines for which plain text files the +appropriate autocommands will be set up in Neovim. The setting is independent +of which Jupytext pairings are active, which is in the metadata for the +`.ipynb` files. All linked files will automatically be kept in sync. Likewise, +when editing `.ipynb` files directly, _all_ linked files will be kept in sync +automatically if `autosync=true`, irrespective of `sync_pattern`. + + +`autosync` *jupytext-autosync* + +If true (recommended), enable automatic synchronization for files paired via +the Jupytext plugin (the plugin for Jupyter Lab). For `.ipynb` files, this +checks if the notebook is paired to any plain text files. If so, it will call +`jupytext --sync` before loading the file and after saving it, to ensure all +files are being kept in sync. It will also periodically check whether the +`.ipynb` file has changed on disk and reload it if necessary, setting the +|`autoread`| option for the current buffer. + +Editing paired files while `autosync = false` will unpair them. + + +============================================================================== +5. Usage *jupytext-usage* + +When opening an `.ipynb` file, this plugin will inject itself into the loading +process and convert the `json` data in the file to a plain text format by +piping it through `jupytext` using the `format` set in |jupytext-options|. + +On saving, the original `.ipynb` file will be updated by piping the content of +the current buffer back into `jupytext`. With the default `update` setting, +this will keep existing outputs and metadata in the notebook. + + +============================================================================== +6. Paired Files *jupytext-paired-files* + +While the Jupytext project provides the command line utility `jupytext` used +by this plugin for on-the-fly conversion between `.ipynb` and plain text +formats, its _primary_ purpose is to provide a plugin for Jupyter to _pair_ +`.ipynb` files with one or more text files that are easier to manage in +version control. + +The intent of the original `jupytext.vim` plugin was to edit `.ipynb` files +_not_ paired in such a way, and not loaded in an active Jupyter session. With +this rewritten version of the plugin, `jupytext.nvim` now supports editing +`.ipynb` files with Neovim if they are paired in Jupyter, and, at least in +principle, even while the notebooks are actively loaded in a running Jupyter +server. + +For this to work, the `autosync` option must be set to `true` (default, see +|jupytext-options|). This automatically handles the update of any paired files +and watches for modifications of the file on disk while it is being edited. + + +============================================================================== +7. History *jupytext-history* + + +v.0.1.0-dev (unreleased) ~ + +- Initial release after complete rewrite from `jupytext.vim` +- Avoid the use of temporary files in the same folder as the `.ipynb` files, + clashing with paired scripts +- Added support for obtaining the notebook language from its metadata, + enabling use of the "script" format +- Added support for automatic synchronization with paired files. + +vim:tw=78:ts=8:noet:ft=help:norl: diff --git a/lua/jupytext.lua b/lua/jupytext.lua new file mode 100644 index 0000000..88448f2 --- /dev/null +++ b/lua/jupytext.lua @@ -0,0 +1,433 @@ +local M = {} + +-- Get the filetype that should be set for the buffer after loading ipynb file +function M.get_filetype(ipynb_file, format, metadata) + if format == 'markdown' then + return format + elseif format:sub(1, 2) == 'md' then + return 'markdown' + elseif format:sub(1, 3) == 'Rmd' then + return 'markdown' + else + return metadata.kernelspec.language + end +end + +-- Plugin options +M.opts = { + jupytext = 'jupytext', + format = 'markdown', + update = true, + filetype = M.get_filetype, + sync_patterns = { '*.md', '*.py', '*.jl', '*.R', '*.Rmd', '*.qmd' }, + autosync = true, + async_write = true, -- undocumented (for testing) + -- TODO: `enabled`, to switch on-/off per buffer +} + +M.setup = function(opts) + for key, value in pairs(opts) do + M.opts[key] = value + end + + local augroup = vim.api.nvim_create_augroup('JupytextPlugin', { clear = true }) + + vim.api.nvim_create_autocmd('BufReadCmd', { + + pattern = '*.ipynb', + group = augroup, + + callback = function(args) + local ipynb_file = args.file -- may be relative path + local bufnr = args.buf + local metadata = M.open_notebook(ipynb_file, bufnr) + vim.b.mtime = vim.uv.fs_stat(ipynb_file).mtime + -- Local autocommands to handle two-way sync + local buf_augroup = 'JupytextPlugin' .. bufnr + vim.api.nvim_create_augroup(buf_augroup, { clear = true }) + + vim.api.nvim_create_autocmd('BufWriteCmd', { + buffer = bufnr, + group = buf_augroup, + callback = function(bufargs) + if bufargs.file:sub(-6) == '.ipynb' then + M.write_notebook(bufargs.file, metadata, bufnr) + else -- write without conversion + local success = M.write_buffer(bufargs.file, bufnr) + if success and (vim.o.cpoptions:find('%+') ~= nil) then + vim.api.nvim_set_option_value('modified', false, { buf = bufnr }) + end + end + end, + }) + + if M.get_option('autosync') and M.is_paired(metadata) then + vim.api.nvim_buf_set_option(bufnr, 'autoread', true) + -- We need autoread to be true, because every save will trigger an + -- update event from the `.ipynb` file being rewritten in the + -- background. + vim.api.nvim_create_autocmd('CursorHold', { + buffer = bufnr, + group = buf_augroup, + callback = function() + vim.api.nvim_command('checktime') + end, + }) + end + end, + }) + + -- autocommands for plain text files + if M.get_option('autosync') and (#M.opts.sync_patterns > 0) then + vim.api.nvim_create_autocmd('CursorHold', { + pattern = M.opts.sync_patterns, + group = augroup, + callback = function() + vim.api.nvim_command('checktime') + end, + }) + + vim.api.nvim_create_autocmd('BufReadPre', { + + pattern = M.opts.sync_patterns, + group = augroup, + + callback = function(args) + local ipynb_file = args.file:match('^(.+)%.%w+$') .. '.ipynb' + if M._file_exists(ipynb_file) then + M.sync(ipynb_file) + print('Synced with "' .. ipynb_file .. '" via jupytext') + end + end, + }) + + vim.api.nvim_create_autocmd('BufWritePost', { + + pattern = M.opts.sync_patterns, + group = augroup, + + callback = function(args) + local ipynb_file = args.file:match('^(.+)%.%w+$') .. '.ipynb' + if M._file_exists(ipynb_file) then + M.sync(ipynb_file, true) -- asynchronous + end + end, + }) + end +end + +function M.get_option(name) + local var_name = 'jupytext_' .. name + if vim.b[var_name] ~= nil then + return vim.b[var_name] + elseif vim.g[name] ~= nil then + return vim.g[var_name] + else + return M.opts[name] + end +end + +function M.schedule(f) + if M.opts.async_write then + vim.schedule(f) + else + f() + end +end + +-- Load ipynb file into the buffer via jupytext conversion +function M.open_notebook(ipynb_file, bufnr) + local source_file = vim.fn.fnamemodify(ipynb_file, ':p') -- absolute path + bufnr = bufnr or 0 -- current buffer, by default + print('Loading via jupytext…') + local metadata = M.get_metadata(source_file) + local autosync = M.get_option('autosync') + if autosync and M.is_paired(metadata) then + M.sync(source_file) + end + local format = M.get_option('format') + local jupytext = M.get_option('jupytext') + if type(format) == 'function' then + format = format(source_file, metadata) + end + local cmd = { jupytext, '--from', 'ipynb', '--to', format, '--output', '-', source_file } + -- TODO: extra --opt arguments, depending on settings + local proc = vim.system(cmd, { text = true }):wait() + if proc.code == 0 then + local text = proc.stdout:gsub('\n$', '') -- strip trailing newline + vim.api.nvim_buf_set_lines(bufnr, -2, -1, false, vim.split(text, '\n')) + local filetype = M.get_option('filetype') + if type(filetype) == 'function' then + filetype = filetype(source_file, format, metadata) + end + vim.api.nvim_set_option_value('filetype', filetype, { buf = bufnr }) + vim.api.nvim_set_option_value('modified', false, { buf = bufnr }) + else + vim.notify(proc.stderr, vim.log.levels.ERROR) + end + print('"' .. ipynb_file .. '" via jupytext with format: ' .. format) + vim.cmd('redraw') + return metadata +end + +-- Call `jupytext --sync` or `jupytext --set-formats` for the given ipynb file +function M.sync(ipynb_file, asynchronous, formats) + local jupytext = M.get_option('jupytext') + local cmd + if formats then + cmd = { jupytext, '--set-formats', formats, ipynb_file } + else + cmd = { jupytext, '--sync', ipynb_file } + end + local function on_exit(proc) + if proc.code ~= 0 then + vim.schedule(function() + vim.notify(proc.stderr, vim.log.levels.ERROR) + end) + end + end + if asynchronous then + vim.system(cmd, { text = true }, on_exit) + else + local proc = vim.system(cmd, { text = true }):wait() + on_exit(proc) + end +end + +-- Write buffer to .ipynb file via jupytext conversion +function M.write_notebook(ipynb_file, metadata, bufnr) + -- ipynb_file may be relative path + local target_file = vim.fn.fnamemodify(ipynb_file, ':p') -- absolute path + local buf_file = vim.api.nvim_buf_get_name(bufnr) -- absolute path + local write_in_place = (target_file == buf_file) + local buf_mtime = vim.b.mtime + local stat = vim.uv.fs_stat(target_file) + if write_in_place then + if stat and stat.mtime.sec ~= buf_mtime.sec then + vim.notify('WARNING: The file has been changed since reading it!!!', vim.log.levels.WARN) + vim.notify('Do you really want to write to it (y/n)? ', vim.log.levels.INFO) + local input = vim.fn.getchar() + local key = vim.fn.nr2char(input) + if key ~= 'y' then + vim.notify('Aborted', vim.log.levels.INFO) + return + end + end + end + local target_is_new = not (stat and stat.type == 'file') + local has_cpo_plus = vim.o.cpoptions:find('%+') ~= nil + metadata = metadata or {} + bufnr = bufnr or 0 -- current buffer, by default + local update = M.get_option('update') + local via_tempfile = update + local autosync = M.get_option('autosync') + local jupytext = M.get_option('jupytext') + local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false) + local cmd = { jupytext, '--to', 'ipynb', '--output', ipynb_file } + if update then + table.insert(cmd, '--update') + end + local formats = M.is_paired(metadata) + local cmd_opts = {} + local tempdir = nil + if via_tempfile then + tempdir = vim.fn.tempname() + vim.fn.mkdir(tempdir) + local yaml_lines = M.get_yaml_lines(lines) + local yaml_data = M.parse_yaml(yaml_lines) + local extension = yaml_data.jupyter.jupytext.text_representation.extension + local basename = ipynb_file:match('([^/]+)%.%w+$') + local tempfile = tempdir .. '/' .. basename .. extension + M.write_buffer(tempfile, bufnr) + table.insert(cmd, tempfile) + else + cmd_opts.stdin = lines + end + local async_write = M.get_option('async_write') + local on_convert = function(proc) + if proc.code == 0 then + local msg = '"' .. ipynb_file .. '"' + if target_is_new then + msg = msg .. ' [New]' + end + msg = msg .. ' ' .. #lines .. 'L via jupytext [w]' + print(msg) + if write_in_place or has_cpo_plus then + M.schedule(function() + vim.api.nvim_set_option_value('modified', false, { buf = bufnr }) + if write_in_place then + vim.b.mtime = vim.uv.fs_stat(ipynb_file).mtime + end + end) + end + if autosync and write_in_place and formats then + M.sync(ipynb_file, true, formats) + -- without autosync, the written file will be unpaired + end + else + M.schedule(function() + vim.notify(proc.stderr, vim.log.levels.ERROR) + end) + end + if tempdir then + M.schedule(function() + vim.fn.delete(tempdir, 'rf') + end) + end + end + if async_write then + vim.system(cmd, cmd_opts, on_convert) + else + local proc = vim.system(cmd, cmd_opts):wait() + on_convert(proc) + end +end + +-- Write buffer to file "as-is" +function M.write_buffer(file, bufnr) + bufnr = bufnr or 0 -- current buffer, by default + local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false) + local fh = io.open(file, 'w') + if fh then + for _, line in ipairs(lines) do + fh:write(line, '\n') + end + fh:close() + return true + else + error('Failed to open file for writing') + return false + end +end + +function M._file_exists(path) + local stat = vim.uv.fs_stat(path) + return stat and stat.type == 'file' +end + +-- Read the metadata from the given ipynb file +function M.get_metadata(ipynb_file) + local success, metadata = pcall(function() + local cmd = { 'jq', '--compact-output', '--monochrome-output', '.metadata', ipynb_file } + local proc = vim.system(cmd, { text = true }):wait() + if proc.code == 0 then + return vim.json.decode(proc.stdout) + else + error('Command exited with non-zero code: ' .. proc.code) + end + end) + if not success then + metadata = M.get_json(ipynb_file).metadata + end + return metadata +end + +function M.get_json(ipynb_file) + local file = io.open(ipynb_file, 'r') + if not file then + error('Could not open file: ' .. ipynb_file) + end + local content = file:read('*all') + file:close() + return vim.json.decode(content) +end + +-- Does metadata indicate that underlying notebook is paired? +-- In non-boolean context, get the paired formats spec +function M.is_paired(metadata) + if metadata.jupytext then + return metadata.jupytext.formats + end + return false +end + +function M.get_yaml_lines(lines) + if type(lines) == 'number' then + local bufnr = lines -- get_yaml_lines(0) does the current buffer + lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false) + end + local yaml_lines = {} + local line_nr = 1 + local first_line = lines[line_nr] + local delimiters = { + ['# ---'] = { '# ', '' }, + ['---'] = { '', '' }, + ['// ---'] = { '// ', '' }, + [';; ---'] = { ';; ', '' }, + ['% ---'] = { '% ', '' }, + ['/ ---'] = { '/ ', '' }, + ['-- ---'] = { '-- ', '' }, + ['(* ---'] = { '(* ', ' *)' }, + ['/* ---'] = { '/* ', ' */' }, + } + local prefix = nil + local suffix = nil + for yaml_start, delims in pairs(delimiters) do + if first_line:sub(1, #yaml_start) == yaml_start then + prefix = delims[1] + suffix = delims[2] + break + end + end + if prefix == nil or suffix == nil then + error('Invalid YAML block') + return {} + end + while line_nr < #lines do + line_nr = line_nr + 1 + local line = lines[line_nr]:sub(#prefix + 1) + if suffix ~= '' then + line = line:sub(1, -#suffix) + end + if line == '---' then + break + else + table.insert(yaml_lines, line) + end + end + return yaml_lines +end + +-- limited YAML parser for the subset of YAML that will appear in the metadata +-- YAML header generated by jupytext +function M.parse_yaml(lines) + local result_table = {} + local stack = {} + local current_indent = '' + + for _, line in ipairs(lines) do + local leading_spaces = line:match('^(%s*)') + local trimmed_line = line:match('^%s*(.-)%s*$') + + if #leading_spaces < #current_indent then + table.remove(stack) + end + current_indent = leading_spaces + + if trimmed_line:sub(-1) == ':' then + local sub_table_name = trimmed_line:sub(1, -2) + table.insert(stack, sub_table_name) + else + local key, value = trimmed_line:match('^(%S+):%s*(.+)$') + if value:sub(1, 1) == "'" and value:sub(-1) == "'" then + value = value:sub(2, -2) + end + local current_subtable = result_table + for _, k in ipairs(stack) do + current_subtable[k] = current_subtable[k] or {} + current_subtable = current_subtable[k] + end + current_subtable[key] = value + end + end + + return result_table +end + +function M.get_yamldata(bufnr) + bufnr = bufnr or 0 -- current buffer, by default + local lines = M.get_yaml_lines(bufnr) + return M.parse_yaml(lines) +end + +return M diff --git a/lua/jupytext/health.lua b/lua/jupytext/health.lua new file mode 100644 index 0000000..6ea1dc6 --- /dev/null +++ b/lua/jupytext/health.lua @@ -0,0 +1,19 @@ +local M = {} + +M.check = function() + vim.health.start('jupytext.nvim') + local proc = vim.system({ 'jupytext', '--version' }):wait() + if proc.code == 0 then + vim.health.ok('jupytext is available') + else + vim.health.error('Jupytext is not available', 'Install jupytext via `pip install jupytext`') + end + proc = vim.system({ 'jq', '--version' }):wait() + if proc.code == 0 then + vim.health.ok('jq is available') + else + vim.health.info('jq is not available') + end +end + +return M diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000..c2e8e12 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e + +mkdir -p ".testenv/config/nvim" +mkdir -p ".testenv/data/nvim" +mkdir -p ".testenv/state/nvim" +mkdir -p ".testenv/run/nvim" +mkdir -p ".testenv/cache/nvim" +PLUGINS=".testenv/data/nvim/site/pack/plugins/start" + +if [ ! -e "$PLUGINS/plenary.nvim" ]; then + git clone --depth=1 https://github.com/nvim-lua/plenary.nvim.git "$PLUGINS/plenary.nvim" +else + (cd "$PLUGINS/plenary.nvim" && git pull) +fi + +XDG_CONFIG_HOME=".testenv/config" \ + XDG_DATA_HOME=".testenv/data" \ + XDG_STATE_HOME=".testenv/state" \ + XDG_RUNTIME_DIR=".testenv/run" \ + XDG_CACHE_HOME=".testenv/cache" \ + nvim --headless -u tests/minimal_init.lua \ + -c "PlenaryBustedDirectory ${1-tests} { minimal_init = './tests/minimal_init.lua', sequential = false }" +echo "Success" diff --git a/tests/minimal_init.lua b/tests/minimal_init.lua new file mode 100644 index 0000000..76aada0 --- /dev/null +++ b/tests/minimal_init.lua @@ -0,0 +1,4 @@ +vim.opt.runtimepath:append('.') + +vim.o.swapfile = false +vim.bo.swapfile = false diff --git a/tests/notebooks/julia.ipynb b/tests/notebooks/julia.ipynb new file mode 100644 index 0000000..09a7b8e --- /dev/null +++ b/tests/notebooks/julia.ipynb @@ -0,0 +1,127 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c7a7bade", + "metadata": {}, + "source": [ + "# Paired Notebook (Julia)" + ] + }, + { + "cell_type": "markdown", + "id": "95898991", + "metadata": {}, + "source": [ + "This is a paired notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "0a918898-20eb-425a-b754-af2b1518e2c2", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-16T09:19:43.137", + "iopub.status.busy": "2024-12-16T09:19:42.935", + "iopub.status.idle": "2024-12-16T09:19:43.425", + "shell.execute_reply": "2024-12-16T09:19:43.411" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "460b41fb", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-16T09:19:43.425", + "iopub.status.busy": "2024-12-16T09:19:43.425", + "iopub.status.idle": "2024-12-16T09:19:43.426", + "shell.execute_reply": "2024-12-16T09:19:43.426" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x + 1" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "972293e8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-16T09:19:43.426", + "iopub.status.busy": "2024-12-16T09:19:43.426", + "iopub.status.idle": "2024-12-16T09:19:43.432", + "shell.execute_reply": "2024-12-16T09:19:43.432" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello World" + ] + } + ], + "source": [ + "print(\"Hello World\")" + ] + }, + { + "cell_type": "markdown", + "id": "d6a3eecd", + "metadata": {}, + "source": [ + "It should have `jupytext` metadata." + ] + } + ], + "metadata": { + "jupytext": { + "formats": "ipynb,jl:percent" + }, + "kernelspec": { + "display_name": "Julia 1.11.2", + "language": "julia", + "name": "julia-1.11" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/notebooks/julia.jl b/tests/notebooks/julia.jl new file mode 100644 index 0000000..af1fe18 --- /dev/null +++ b/tests/notebooks/julia.jl @@ -0,0 +1,32 @@ +# --- +# jupyter: +# jupytext: +# formats: ipynb,jl:percent +# text_representation: +# extension: .jl +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.16.4 +# kernelspec: +# display_name: Julia 1.11.2 +# language: julia +# name: julia-1.11 +# --- + +# %% [markdown] +# # Paired Notebook (Julia) + +# %% [markdown] +# This is a paired notebook + +# %% +x = 1 + +# %% +x + 1 + +# %% +print("Hello World") + +# %% [markdown] +# It should have `jupytext` metadata. diff --git a/tests/notebooks/paired.ipynb b/tests/notebooks/paired.ipynb new file mode 100644 index 0000000..eea615e --- /dev/null +++ b/tests/notebooks/paired.ipynb @@ -0,0 +1,133 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2e827ebb-5d8a-491f-9e62-c680be25c27a", + "metadata": {}, + "source": [ + "# Paired Notebook" + ] + }, + { + "cell_type": "markdown", + "id": "cfb8907f-324c-408a-8a8b-a7ff6340d3be", + "metadata": {}, + "source": [ + "This is a paired notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "404894a9-4a0c-4d27-a869-981b5ddb34ea", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-16T13:46:25.015919Z", + "iopub.status.busy": "2024-12-16T13:46:25.014031Z", + "iopub.status.idle": "2024-12-16T13:46:25.021865Z", + "shell.execute_reply": "2024-12-16T13:46:25.021488Z", + "shell.execute_reply.started": "2024-12-16T13:46:25.015871Z" + } + }, + "outputs": [], + "source": [ + "x = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "5d1ddd99-f307-4184-81b0-6265f23abf0e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-16T13:46:25.022551Z", + "iopub.status.busy": "2024-12-16T13:46:25.022408Z", + "iopub.status.idle": "2024-12-16T13:46:25.026843Z", + "shell.execute_reply": "2024-12-16T13:46:25.026542Z", + "shell.execute_reply.started": "2024-12-16T13:46:25.022533Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "2" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x + 1" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d0236586-bb9e-4d24-a46d-2d67dba67ec9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-16T13:46:25.027473Z", + "iopub.status.busy": "2024-12-16T13:46:25.027379Z", + "iopub.status.idle": "2024-12-16T13:46:25.029438Z", + "shell.execute_reply": "2024-12-16T13:46:25.029048Z", + "shell.execute_reply.started": "2024-12-16T13:46:25.027463Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello World\n" + ] + } + ], + "source": [ + "print(\"Hello World\")" + ] + }, + { + "cell_type": "markdown", + "id": "0041a9aa-0708-498c-80e6-db985095c588", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-16T13:46:25.029914Z", + "iopub.status.busy": "2024-12-16T13:46:25.029842Z", + "iopub.status.idle": "2024-12-16T13:46:25.032547Z", + "shell.execute_reply": "2024-12-16T13:46:25.031931Z", + "shell.execute_reply.started": "2024-12-16T13:46:25.029906Z" + } + }, + "source": [ + "It should have `jupytext` metadata." + ] + } + ], + "metadata": { + "jupytext": { + "formats": "ipynb,py:light,md:myst" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/notebooks/paired.md b/tests/notebooks/paired.md new file mode 100644 index 0000000..14891a3 --- /dev/null +++ b/tests/notebooks/paired.md @@ -0,0 +1,33 @@ +--- +jupytext: + formats: ipynb,py:light,md:myst + text_representation: + extension: .md + format_name: myst + format_version: 0.13 + jupytext_version: 1.16.4 +kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + +# Paired Notebook + ++++ + +This is a paired notebook + +```{code-cell} ipython3 +x = 1 +``` + +```{code-cell} ipython3 +x + 1 +``` + +```{code-cell} ipython3 +print("Hello World") +``` + +It should have `jupytext` metadata. diff --git a/tests/notebooks/paired.py b/tests/notebooks/paired.py new file mode 100644 index 0000000..f010ad6 --- /dev/null +++ b/tests/notebooks/paired.py @@ -0,0 +1,26 @@ +# --- +# jupyter: +# jupytext: +# formats: ipynb,py:light,md:myst +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.16.4 +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# # Paired Notebook + +# This is a paired notebook + +x = 1 + +x + 1 + +print("Hello World") + +# It should have `jupytext` metadata. diff --git a/tests/notebooks/unpaired.ipynb b/tests/notebooks/unpaired.ipynb new file mode 100644 index 0000000..7f6e947 --- /dev/null +++ b/tests/notebooks/unpaired.ipynb @@ -0,0 +1,93 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0b0ddd66-850b-4e10-a3f3-ed26a1b45ec7", + "metadata": {}, + "source": [ + "# Unpaired Notebook" + ] + }, + { + "cell_type": "markdown", + "id": "56671a89-4b1f-46f4-b127-91847ddb7a35", + "metadata": {}, + "source": [ + "This is an unpaired notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "869ddff4-2b70-43d7-9f1f-3dd7188fe246", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-05T19:10:42.630105Z", + "iopub.status.busy": "2024-12-05T19:10:42.629675Z", + "iopub.status.idle": "2024-12-05T19:10:42.635743Z", + "shell.execute_reply": "2024-12-05T19:10:42.634611Z", + "shell.execute_reply.started": "2024-12-05T19:10:42.630077Z" + } + }, + "outputs": [], + "source": [ + "x = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b6ef69ce-616f-421e-849a-b1ffcb6d4c10", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-05T20:23:38.327491Z", + "iopub.status.busy": "2024-12-05T20:23:38.326909Z", + "iopub.status.idle": "2024-12-05T20:23:38.335909Z", + "shell.execute_reply": "2024-12-05T20:23:38.333672Z", + "shell.execute_reply.started": "2024-12-05T20:23:38.327456Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello World\n" + ] + } + ], + "source": [ + "print(\"Hello World\")" + ] + }, + { + "cell_type": "markdown", + "id": "12302f49-cac5-45b0-b7e3-d5b50d217278", + "metadata": {}, + "source": [ + "It should not have `jupytext` metadata." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/paired_julia_spec.lua b/tests/paired_julia_spec.lua new file mode 100644 index 0000000..b9ad251 --- /dev/null +++ b/tests/paired_julia_spec.lua @@ -0,0 +1,32 @@ +local util = require('tests.test_util') + +describe('a paired .ipynb julia file', function() + local notebooks = util.notebooks() + print(notebooks) + local ipynb_file = notebooks .. 'julia.ipynb' + + it('has jupytext metadata', function() + local metadata = require('jupytext').get_metadata(ipynb_file) + assert.is_truthy(metadata.jupytext) + end) + + it('can be loaded with automatic filetype "python"', function() + require('jupytext').setup({ format = 'script' }) + vim.cmd('edit ' .. ipynb_file) + local lines = vim.api.nvim_buf_get_lines(0, 0, -1, false) + assert.are.same(lines[1], '# ---') -- YAML header, not JSON + assert.are.same(vim.bo.filetype, 'julia') + end) + + it('can be edited and retain outputs', function() + require('jupytext').setup({ format = 'script', async_write = false }) + vim.cmd('edit ' .. ipynb_file) + vim.cmd('%s/World//g') + vim.cmd.write() + local json = require('jupytext').get_json(ipynb_file) + assert.is_nil(json.metadata.jupytext) + assert.is_truthy(json.cells[5].source[1]:find('Hello')) + assert.is_nil(json.cells[5].source[1]:find('World')) + assert.is_truthy(json.cells[5].outputs) + end) +end) diff --git a/tests/paired_python_spec.lua b/tests/paired_python_spec.lua new file mode 100644 index 0000000..00a3ffc --- /dev/null +++ b/tests/paired_python_spec.lua @@ -0,0 +1,32 @@ +local util = require('tests.test_util') + +describe('a paired .ipynb python file', function() + local notebooks = util.notebooks() + print(notebooks) + local ipynb_file = notebooks .. 'paired.ipynb' + + it('has jupytext metadata', function() + local metadata = require('jupytext').get_metadata(ipynb_file) + assert.is_truthy(metadata.jupytext) + end) + + it('can be loaded with automatic filetype "python"', function() + require('jupytext').setup({ format = 'script' }) + vim.cmd('edit ' .. ipynb_file) + local lines = vim.api.nvim_buf_get_lines(0, 0, -1, false) + assert.are.same(lines[1], '# ---') -- YAML header, not JSON + assert.are.same(vim.bo.filetype, 'python') + end) + + it('can be edited and retain outputs', function() + require('jupytext').setup({ format = 'script', async_write = false }) + vim.cmd('edit ' .. ipynb_file) + vim.cmd('%s/World//g') + vim.cmd.write() + local json = require('jupytext').get_json(ipynb_file) + assert.is_nil(json.metadata.jupytext) + assert.is_truthy(json.cells[5].source[1]:find('Hello')) + assert.is_nil(json.cells[5].source[1]:find('World')) + assert.is_truthy(json.cells[5].outputs) + end) +end) diff --git a/tests/test_util.lua b/tests/test_util.lua new file mode 100644 index 0000000..67823d7 --- /dev/null +++ b/tests/test_util.lua @@ -0,0 +1,23 @@ +local M = {} + +M.notebooks = function(opts) + local tempdir = vim.fn.tempname() .. '/' + vim.fn.mkdir(tempdir, 'p') + local current_dir = debug.getinfo(1, 'S').source:match('@(.*)'):match('(.*/)') + local notebooks_dir = current_dir .. 'notebooks' + for _, filename in ipairs(vim.fn.readdir(notebooks_dir)) do + file = notebooks_dir .. '/' .. filename + local cp_cmd = { 'cp', file, tempdir } + local proc = vim.system(cp_cmd):wait() + if proc.code ~= 0 then + print('ERROR: ' .. proc.stderr) + end + end + if opts and opts.debug then + proc = vim.system({ 'tree', '-a', tempdir }):wait() + print(proc.stdout) + end + return tempdir +end + +return M diff --git a/tests/unpaired_spec.lua b/tests/unpaired_spec.lua new file mode 100644 index 0000000..61e232e --- /dev/null +++ b/tests/unpaired_spec.lua @@ -0,0 +1,31 @@ +local util = require('tests.test_util') + +describe('an unpaired .ipynb file', function() + local notebooks = util.notebooks() + print(notebooks) + local ipynb_file = notebooks .. 'unpaired.ipynb' + + it('does not have jupytext metadata', function() + local metadata = require('jupytext').get_metadata(ipynb_file) + assert.is_nil(metadata.jupytext) + end) + + it('can be loaded', function() + require('jupytext').setup({ format = 'markdown' }) + vim.cmd('edit ' .. ipynb_file) + local lines = vim.api.nvim_buf_get_lines(0, 0, -1, false) + assert.are.same(lines[1], '---') -- YAML header, not JSON + end) + + it('can be edited and retain outputs', function() + require('jupytext').setup({ format = 'markdown', async_write = false }) + vim.cmd('edit ' .. ipynb_file) + vim.cmd('%s/World//g') + vim.cmd.write() + local json = require('jupytext').get_json(ipynb_file) + assert.is_nil(json.metadata.jupytext) + assert.is_truthy(json.cells[4].source[1]:find('Hello')) + assert.is_nil(json.cells[4].source[1]:find('World')) + assert.is_truthy(json.cells[4].outputs) + end) +end)