Skip to content

Commit de0e1c9

Browse files
toyobayashicclauss
toyobayashi
and
cclauss
authored
fix: support cross compiling for wasm with make generator (#222)
* fix: support cross compiling for wasm with make generator * fix: lint * refactor for readability Co-authored-by: Christian Clauss <cclauss@me.com> * replace separator in make generator on Windows * snake_case * found more place to replace sep * lint * replace sep in compiler path * fix sed unterminated `s' command error on Windows * path includes `\` so replace the ended `\` only * replace `\` with `/` in depfile on win * lint * fix: trailing `\` in raw string * revert: flavor can be set via `-f make-linux` so no need to change the mac params * fix: also do not use raw string in windows branch due to trailing `\` break editor highlight * fix: respect user specified AR_target environment variable * feat: detect wasm flavor * lint: Too many return statements * fix get compiler predefines on windows * GetCrossCompilerPredefines always return dict * do not broad exceptions * test: GetCrossCompilerPredefines * fix lint * refactor: do not put so many lines in try block * fix: finally block should wait until subprocess terminate * suggestion change --------- Co-authored-by: Christian Clauss <cclauss@me.com>
1 parent 7b20b46 commit de0e1c9

File tree

3 files changed

+172
-27
lines changed

3 files changed

+172
-27
lines changed

‎pylib/gyp/common.py

Lines changed: 56 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -422,17 +422,58 @@ def EnsureDirExists(path):
422422
except OSError:
423423
pass
424424

425+
def GetCrossCompilerPredefines(): # -> dict
426+
cmd = []
427+
if CC := os.environ.get("CC_target") or os.environ.get("CC"):
428+
cmd += CC.split(" ")
429+
if CFLAGS := os.environ.get("CFLAGS"):
430+
cmd += CFLAGS.split(" ")
431+
elif CXX := os.environ.get("CXX_target") or os.environ.get("CXX"):
432+
cmd += CXX.split(" ")
433+
if CXXFLAGS := os.environ.get("CXXFLAGS"):
434+
cmd += CXXFLAGS.split(" ")
435+
else:
436+
return {}
425437

426-
def GetFlavor(params):
438+
if sys.platform == "win32":
439+
fd, input = tempfile.mkstemp(suffix=".c")
440+
try:
441+
os.close(fd)
442+
out = subprocess.Popen(
443+
[*cmd, "-dM", "-E", "-x", "c", input],
444+
shell=True,
445+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT
446+
)
447+
stdout = out.communicate()[0]
448+
finally:
449+
os.unlink(input)
450+
else:
451+
input = "/dev/null"
452+
out = subprocess.Popen(
453+
[*cmd, "-dM", "-E", "-x", "c", input],
454+
shell=False,
455+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT
456+
)
457+
stdout = out.communicate()[0]
458+
459+
defines = {}
460+
lines = stdout.decode("utf-8").replace("\r\n", "\n").split("\n")
461+
for line in lines:
462+
if not line:
463+
continue
464+
define_directive, key, *value = line.split(" ")
465+
assert define_directive == "#define"
466+
defines[key] = " ".join(value)
467+
return defines
468+
469+
def GetFlavorByPlatform():
427470
"""Returns |params.flavor| if it's set, the system's default flavor else."""
428471
flavors = {
429472
"cygwin": "win",
430473
"win32": "win",
431474
"darwin": "mac",
432475
}
433476

434-
if "flavor" in params:
435-
return params["flavor"]
436477
if sys.platform in flavors:
437478
return flavors[sys.platform]
438479
if sys.platform.startswith("sunos"):
@@ -452,6 +493,18 @@ def GetFlavor(params):
452493

453494
return "linux"
454495

496+
def GetFlavor(params):
497+
if "flavor" in params:
498+
return params["flavor"]
499+
500+
defines = GetCrossCompilerPredefines()
501+
if "__EMSCRIPTEN__" in defines:
502+
return "emscripten"
503+
if "__wasm__" in defines:
504+
return "wasi" if "__wasi__" in defines else "wasm"
505+
506+
return GetFlavorByPlatform()
507+
455508

456509
def CopyTool(flavor, out_path, generator_flags={}):
457510
"""Finds (flock|mac|win)_tool.gyp in the gyp directory and copies it

‎pylib/gyp/common_test.py

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
import gyp.common
1010
import unittest
1111
import sys
12-
12+
import os
13+
import subprocess
14+
from unittest.mock import patch, MagicMock
1315

1416
class TestTopologicallySorted(unittest.TestCase):
1517
def test_Valid(self):
@@ -73,6 +75,62 @@ def test_platform_default(self):
7375
def test_param(self):
7476
self.assertFlavor("foobar", "linux2", {"flavor": "foobar"})
7577

78+
class MockCommunicate:
79+
def __init__(self, stdout):
80+
self.stdout = stdout
81+
82+
def decode(self, encoding):
83+
return self.stdout
84+
85+
@patch("os.close")
86+
@patch("os.unlink")
87+
@patch("tempfile.mkstemp")
88+
def test_GetCrossCompilerPredefines(self, mock_mkstemp, mock_unlink, mock_close):
89+
mock_close.return_value = None
90+
mock_unlink.return_value = None
91+
mock_mkstemp.return_value = (0, "temp.c")
92+
93+
def mock_run(env, defines_stdout):
94+
with patch("subprocess.Popen") as mock_popen:
95+
mock_process = MagicMock()
96+
mock_process.communicate.return_value = (
97+
TestGetFlavor.MockCommunicate(defines_stdout), None)
98+
mock_process.stdout = MagicMock()
99+
mock_popen.return_value = mock_process
100+
expected_input = "temp.c" if sys.platform == "win32" else "/dev/null"
101+
with patch.dict(os.environ, env):
102+
defines = gyp.common.GetCrossCompilerPredefines()
103+
flavor = gyp.common.GetFlavor({})
104+
if env.get("CC_target"):
105+
mock_popen.assert_called_with(
106+
[env["CC_target"], "-dM", "-E", "-x", "c", expected_input],
107+
shell=sys.platform == "win32",
108+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
109+
return [defines, flavor]
110+
111+
[defines1, _] = mock_run({}, "")
112+
self.assertDictEqual({}, defines1)
113+
114+
[defines2, flavor2] = mock_run(
115+
{ "CC_target": "/opt/wasi-sdk/bin/clang" },
116+
"#define __wasm__ 1\n#define __wasi__ 1\n"
117+
)
118+
self.assertDictEqual({ "__wasm__": "1", "__wasi__": "1" }, defines2)
119+
self.assertEqual("wasi", flavor2)
120+
121+
[defines3, flavor3] = mock_run(
122+
{ "CC_target": "/opt/wasi-sdk/bin/clang" },
123+
"#define __wasm__ 1\n"
124+
)
125+
self.assertDictEqual({ "__wasm__": "1" }, defines3)
126+
self.assertEqual("wasm", flavor3)
127+
128+
[defines4, flavor4] = mock_run(
129+
{ "CC_target": "/emsdk/upstream/emscripten/emcc" },
130+
"#define __EMSCRIPTEN__ 1\n"
131+
)
132+
self.assertDictEqual({ "__EMSCRIPTEN__": "1" }, defines4)
133+
self.assertEqual("emscripten", flavor4)
76134

77135
if __name__ == "__main__":
78136
unittest.main()

‎pylib/gyp/generator/make.py

Lines changed: 57 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import os
2626
import re
2727
import subprocess
28+
import sys
2829
import gyp
2930
import gyp.common
3031
import gyp.xcode_emulation
@@ -378,7 +379,7 @@ def CalculateGeneratorInputInfo(params):
378379
CXXFLAGS.target ?= $(CPPFLAGS) $(CXXFLAGS)
379380
LINK.target ?= %(LINK.target)s
380381
LDFLAGS.target ?= $(LDFLAGS)
381-
AR.target ?= $(AR)
382+
AR.target ?= %(AR.target)s
382383
PLI.target ?= %(PLI.target)s
383384
384385
# C++ apps need to be linked with g++.
@@ -442,13 +443,21 @@ def CalculateGeneratorInputInfo(params):
442443
define fixup_dep
443444
# The depfile may not exist if the input file didn't have any #includes.
444445
touch $(depfile).raw
445-
# Fixup path as in (1).
446-
sed -e "s|^$(notdir $@)|$@|" $(depfile).raw >> $(depfile)
446+
# Fixup path as in (1).""" +
447+
(r"""
448+
sed -e "s|^$(notdir $@)|$@|" -re 's/\\\\([^$$])/\/\1/g' $(depfile).raw >> $(depfile)"""
449+
if sys.platform == 'win32' else r"""
450+
sed -e "s|^$(notdir $@)|$@|" $(depfile).raw >> $(depfile)""") +
451+
r"""
447452
# Add extra rules as in (2).
448453
# We remove slashes and replace spaces with new lines;
449454
# remove blank lines;
450-
# delete the first line and append a colon to the remaining lines.
451-
sed -e 's|\\||' -e 'y| |\n|' $(depfile).raw |\
455+
# delete the first line and append a colon to the remaining lines.""" +
456+
("""
457+
sed -e 's/\\\\\\\\$$//' -e 's/\\\\\\\\/\\//g' -e 'y| |\\n|' $(depfile).raw |\\"""
458+
if sys.platform == 'win32' else """
459+
sed -e 's|\\\\||' -e 'y| |\\n|' $(depfile).raw |\\""") +
460+
r"""
452461
grep -v '^$$' |\
453462
sed -e 1d -e 's|$$|:|' \
454463
>> $(depfile)
@@ -724,6 +733,10 @@ def QuoteIfNecessary(string):
724733
string = '"' + string.replace('"', '\\"') + '"'
725734
return string
726735

736+
def replace_sep(string):
737+
if sys.platform == 'win32':
738+
string = string.replace('\\\\', '/').replace('\\', '/')
739+
return string
727740

728741
def StringToMakefileVariable(string):
729742
"""Convert a string to a value that is acceptable as a make variable name."""
@@ -859,7 +872,7 @@ def Write(
859872
self.output = self.ComputeMacBundleOutput(spec)
860873
self.output_binary = self.ComputeMacBundleBinaryOutput(spec)
861874
else:
862-
self.output = self.output_binary = self.ComputeOutput(spec)
875+
self.output = self.output_binary = replace_sep(self.ComputeOutput(spec))
863876

864877
self.is_standalone_static_library = bool(
865878
spec.get("standalone_static_library", 0)
@@ -985,7 +998,7 @@ def WriteSubMake(self, output_filename, makefile_path, targets, build_dir):
985998
# sub-project dir (see test/subdirectory/gyptest-subdir-all.py).
986999
self.WriteLn(
9871000
"export builddir_name ?= %s"
988-
% os.path.join(os.path.dirname(output_filename), build_dir)
1001+
% replace_sep(os.path.join(os.path.dirname(output_filename), build_dir))
9891002
)
9901003
self.WriteLn(".PHONY: all")
9911004
self.WriteLn("all:")
@@ -2063,7 +2076,7 @@ def WriteList(self, value_list, variable=None, prefix="", quoter=QuoteIfNecessar
20632076
"""
20642077
values = ""
20652078
if value_list:
2066-
value_list = [quoter(prefix + value) for value in value_list]
2079+
value_list = [replace_sep(quoter(prefix + value)) for value in value_list]
20672080
values = " \\\n\t" + " \\\n\t".join(value_list)
20682081
self.fp.write(f"{variable} :={values}\n\n")
20692082

@@ -2369,10 +2382,12 @@ def WriteAutoRegenerationRule(params, root_makefile, makefile_name, build_files)
23692382
"\t$(call do_cmd,regen_makefile)\n\n"
23702383
% {
23712384
"makefile_name": makefile_name,
2372-
"deps": " ".join(SourceifyAndQuoteSpaces(bf) for bf in build_files),
2373-
"cmd": gyp.common.EncodePOSIXShellList(
2374-
[gyp_binary, "-fmake"] + gyp.RegenerateFlags(options) + build_files_args
2385+
"deps": replace_sep(
2386+
" ".join(SourceifyAndQuoteSpaces(bf) for bf in build_files)
23752387
),
2388+
"cmd": replace_sep(gyp.common.EncodePOSIXShellList(
2389+
[gyp_binary, "-fmake"] + gyp.RegenerateFlags(options) + build_files_args
2390+
)),
23762391
}
23772392
)
23782393

@@ -2435,33 +2450,52 @@ def CalculateMakefilePath(build_file, base_name):
24352450
makefile_path = os.path.join(
24362451
options.toplevel_dir, options.generator_output, makefile_name
24372452
)
2438-
srcdir = gyp.common.RelativePath(srcdir, options.generator_output)
2453+
srcdir = replace_sep(gyp.common.RelativePath(srcdir, options.generator_output))
24392454
srcdir_prefix = "$(srcdir)/"
24402455

24412456
flock_command = "flock"
24422457
copy_archive_arguments = "-af"
24432458
makedep_arguments = "-MMD"
2459+
2460+
# wasm-ld doesn't support --start-group/--end-group
2461+
link_commands = LINK_COMMANDS_LINUX
2462+
if flavor in ["wasi", "wasm"]:
2463+
link_commands = link_commands.replace(' -Wl,--start-group', '').replace(
2464+
' -Wl,--end-group', ''
2465+
)
2466+
2467+
CC_target = replace_sep(GetEnvironFallback(("CC_target", "CC"), "$(CC)"))
2468+
AR_target = replace_sep(GetEnvironFallback(("AR_target", "AR"), "$(AR)"))
2469+
CXX_target = replace_sep(GetEnvironFallback(("CXX_target", "CXX"), "$(CXX)"))
2470+
LINK_target = replace_sep(GetEnvironFallback(("LINK_target", "LINK"), "$(LINK)"))
2471+
PLI_target = replace_sep(GetEnvironFallback(("PLI_target", "PLI"), "pli"))
2472+
CC_host = replace_sep(GetEnvironFallback(("CC_host", "CC"), "gcc"))
2473+
AR_host = replace_sep(GetEnvironFallback(("AR_host", "AR"), "ar"))
2474+
CXX_host = replace_sep(GetEnvironFallback(("CXX_host", "CXX"), "g++"))
2475+
LINK_host = replace_sep(GetEnvironFallback(("LINK_host", "LINK"), "$(CXX.host)"))
2476+
PLI_host = replace_sep(GetEnvironFallback(("PLI_host", "PLI"), "pli"))
2477+
24442478
header_params = {
24452479
"default_target": default_target,
24462480
"builddir": builddir_name,
24472481
"default_configuration": default_configuration,
24482482
"flock": flock_command,
24492483
"flock_index": 1,
2450-
"link_commands": LINK_COMMANDS_LINUX,
2484+
"link_commands": link_commands,
24512485
"extra_commands": "",
24522486
"srcdir": srcdir,
24532487
"copy_archive_args": copy_archive_arguments,
24542488
"makedep_args": makedep_arguments,
2455-
"CC.target": GetEnvironFallback(("CC_target", "CC"), "$(CC)"),
2456-
"AR.target": GetEnvironFallback(("AR_target", "AR"), "$(AR)"),
2457-
"CXX.target": GetEnvironFallback(("CXX_target", "CXX"), "$(CXX)"),
2458-
"LINK.target": GetEnvironFallback(("LINK_target", "LINK"), "$(LINK)"),
2459-
"PLI.target": GetEnvironFallback(("PLI_target", "PLI"), "pli"),
2460-
"CC.host": GetEnvironFallback(("CC_host", "CC"), "gcc"),
2461-
"AR.host": GetEnvironFallback(("AR_host", "AR"), "ar"),
2462-
"CXX.host": GetEnvironFallback(("CXX_host", "CXX"), "g++"),
2463-
"LINK.host": GetEnvironFallback(("LINK_host", "LINK"), "$(CXX.host)"),
2464-
"PLI.host": GetEnvironFallback(("PLI_host", "PLI"), "pli"),
2489+
"CC.target": CC_target,
2490+
"AR.target": AR_target,
2491+
"CXX.target": CXX_target,
2492+
"LINK.target": LINK_target,
2493+
"PLI.target": PLI_target,
2494+
"CC.host": CC_host,
2495+
"AR.host": AR_host,
2496+
"CXX.host": CXX_host,
2497+
"LINK.host": LINK_host,
2498+
"PLI.host": PLI_host,
24652499
}
24662500
if flavor == "mac":
24672501
flock_command = "./gyp-mac-tool flock"

0 commit comments

Comments
 (0)