Merge branch 'release-0.6.0' into release

.gitignore

@@ -15,3 +15,4 @@ work/
 output/
 _ReSharper.*
 .vscode/
+src/common/dtrace_probes.h

CHANGELOG.md

@@ -2,6 +2,26 @@
 
 All notable changes to the Pony compiler and standard library will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a CHANGELOG](http://keepachangelog.com/).
 
+## [0.6.0] - 2016-10-20
+
+### Fixed
+
+- Compiling ponyrt with Clang versions >= 3.3, < 3.6.
+- Restrict mutable tuple recovery to maintain reference capability security (issue #1123)
+- Crash in the runtime scheduler queues
+
+### Added
+
+- DTrace and SystemTap support - `use=dtrace`
+
+### Changed
+
+- Replaces `use=telemetry` by DTrace/SystemTap scripts
+- `String.cstring()` now always returns a null-terminated string
+  (which may result in a copy) while `cpointer()` (also available on
+  `Array` objects) returns a pointer to the underlying array as-is
+  (issue #1309).
+
 ## [0.5.1] - 2016-10-15
 
 ### Fixed

Makefile

@@ -81,7 +81,7 @@ ifdef destdir
   endif
 endif
 
-ifeq ($(OSTYPE),osx)
+ifneq (,$(filter $(OSTYPE), osx freebsd))
   symlink.flags = -sf
 else
   symlink.flags = -srf
@@ -124,9 +124,14 @@ ifdef use
     PONY_BUILD_DIR := $(PONY_BUILD_DIR)-pooltrack
   endif
 
-  ifneq (,$(filter $(use), telemetry))
-    ALL_CFLAGS += -DUSE_TELEMETRY
-    PONY_BUILD_DIR := $(PONY_BUILD_DIR)-telemetry
+  ifneq (,$(filter $(use), dtrace))
+    DTRACE ?= $(shell which dtrace)
+    ifeq (, $(DTRACE))
+      $(error No dtrace compatible user application static probe generation tool found)
+    endif
+
+    ALL_CFLAGS += -DUSE_DYNAMIC_TRACE
+    PONY_BUILD_DIR := $(PONY_BUILD_DIR)-dtrace
   endif
 endif
 
@@ -371,6 +376,10 @@ ifeq ($(OSTYPE),linux)
   libponyrt.tests.links += pthread dl
 endif
 
+ifneq (, $(DTRACE))
+  $(shell $(DTRACE) -h -s $(PONY_SOURCE_DIR)/common/dtrace_probes.d -o $(PONY_SOURCE_DIR)/common/dtrace_probes.h)
+endif
+
 # Overwrite the default linker for a target.
 ponyc.linker = $(CXX) #compile as C but link as CPP (llvm)
 
@@ -523,7 +532,17 @@ $(foreach d,$($(1).depends),$(eval depends += $($(d))/$(d).$(LIB_EXT)))
 ifeq ($(1),libponyrt)
 $($(1))/libponyrt.$(LIB_EXT): $(depends) $(ofiles)
 	@echo 'Linking libponyrt'
+    ifneq (,$(DTRACE))
+    ifeq ($(OSTYPE), linux)
+	@echo 'Generating dtrace object file'
+	$(SILENT)$(DTRACE) -G -s $(PONY_SOURCE_DIR)/common/dtrace_probes.d -o $(PONY_BUILD_DIR)/dtrace_probes.o
+	$(SILENT)$(AR) $(AR_FLAGS) $$@ $(ofiles) $(PONY_BUILD_DIR)/dtrace_probes.o
+    else
 	$(SILENT)$(AR) $(AR_FLAGS) $$@ $(ofiles)
+    endif
+    else
+	$(SILENT)$(AR) $(AR_FLAGS) $$@ $(ofiles)
+    endif
   ifeq ($(runtime-bitcode),yes)
 $($(1))/libponyrt.bc: $(depends) $(bcfiles)
 	@echo 'Generating bitcode for libponyrt'
@@ -685,6 +704,7 @@ stats:
 
 clean:
 	@rm -rf $(PONY_BUILD_DIR)
+	@rm src/common/dtrace_probes.h
 	-@rmdir build 2>/dev/null ||:
 	@echo 'Repository cleaned ($(PONY_BUILD_DIR)).'
 
@@ -702,7 +722,7 @@ help:
 	@echo 'USE OPTIONS:'
 	@echo '   valgrind'
 	@echo '   pooltrack'
-	@echo '   telemetry'
+	@echo '   dtrace'
 	@echo
 	@echo 'TARGETS:'
 	@echo '  libponyc          Pony compiler library'

VERSION

@@ -1 +1 @@
-0.5.1
+0.6.0

examples/dtrace/README.md

@@ -0,0 +1,131 @@
+# DTrace in PonyC
+
+[DTrace](http://dtrace.org/guide/preface.html) provides a library for dynamic
+tracing of events. This document describes the implementation of DTrace within
+the Pony compiler. In particular, it focuses on how to use and extend the DTrace
+implementation within Pony.
+
+DTrace is only available for MacOS and BSD. For Linux there is an alternative
+called [SystemTap](https://sourceware.org/systemtap/). SystemTap uses the same
+resources as DTrace, but uses different scripts. You will find more information
+on using SystemTap within the Pony compiler [here](../systemtap/README.md).
+
+## Using DTrace scripts for Pony
+
+You can find various example DTrace scripts in `example/dtrace/`. To run these
+scripts, you can use a command in the form of `[script] -c [binary]`. You need
+to use a PonyC compiler compiled with the DTrace flag to use DTrace. To compile
+a DTrace enabled compiler, you can use the following command: `make
+config=release use=dtrace`.
+
+We can also execute a script using the dtrace command. This is necessary if the
+script does not contain a "Shebang". We can do it using the following command:
+`dtrace -s [script] -c [binary]`.
+
+### MacOS 10.11+
+
+MacOS El Capitan for the first time included "System Integrity Protection". This
+feature does not allow for all DTrace features to be used. This includes custom
+DTrace providers. To get around this problem, the following steps can be
+followed:
+
+1. Boot your Mac into Recovery Mode.
+2. Open the Terminal from the Utilities menu.
+3. Then use the following commands:
+
+```bash
+
+csrutil clear # restore the default configuration first
+
+csrutil enable --without dtrace # disable dtrace restrictions *only*
+
+```
+
+After completing these steps your Mac should be able to use DTrace after a
+restart. For more information visit the following article:
+[link](http://internals.exposed/blog/dtrace-vs-sip.html).
+
+## Writing scripts for DTrace in Pony
+
+The following paragraph will inform you on the Pony provider and its probes for
+DTrace. We refer to the [DTrace
+documentation](http://dtrace.org/guide/preface.html) for more information on the
+syntax and possibilities.
+
+The Pony provider for DTrace consists of DTrace probes implemented in the
+Runtime. A binary compiled with DTrace enabled will allow the user access to the
+information of the probes. You can find a list of all probes and their arguments
+in [`src/common/dtrace_probes.d`](../../src/common/dtrace_probes.d).  The
+following is a toy example DTrace script:
+
+```
+
+pony$target:::gc-start
+{
+  @ = count();
+}
+
+END
+{
+  printa(@);
+}
+
+```
+
+This script increases a counter every time the "GC Start" probe is *fired* and
+prints it result at the end. Note the way in which we access the probe. It
+consists of two parts: the first part is the provider and the second part, after `:::`
+is the name of the probe. The provider is during runtime appended by the process
+ID. In this example we use the `$target` variable to find the process ID.
+Another option is to use `pony*`, but note that this could match many processes.
+The name of the probe are also different from the names in
+[`src/common/dtrace_probes.d`](../../src/common/dtrace_probes.d). In Dtrace
+scripts you have to replace the `__` in the names by `-`.
+
+To make these scripts executable, like the ones in the examples, we use the
+following "Shebang": `#!/usr/bin/env dtrace -s`. Extra options can be added to
+improve its functionality.
+
+## Extending the DTrace implementation in Pony
+
+You can extend the DTrace implementation by adding more probes or extra
+information to existing probes.  All probes are defined in
+[`src/common/dtrace_probes.d`](../../src/common/dtrace_probes.d). Usually their
+names of their module and the event that triggers them. To install Probes in C
+use the macros defined in `src/common/dtrace.h`.  To fire a probe in the C code
+use the macro `DTRACEx`; where `x` is the number of arguments of  the probe.
+There is also a macro `DTRACE_ENABLED`; its use allows for code to be only
+executed when a probe is enabled.
+
+### Adding a probe
+
+The first step to add a probe is to add its definition into
+[`src/common/dtrace_probes.d`](../../src/common/dtrace_probes.d). We have split
+the names of the probes into two parts: its category and the specific event. The
+name is split using `__`. We also add a comment to a probe explaining when it's
+fired and the information contained in its arguments.
+
+After adding the probe, we use it in the C code using the macros. Note that the
+split in the name is only a single underscore in the C code. The name of the
+probe should also be capitalised. We can call the probe defined as `gc__start`
+using the statement `DTRACE1(GC_START,  scheduler)`. In this statement
+`scheduler` is the data used as the first argument.
+
+Then once the probe has been placed in all the appropriate places, we are ready
+to recompile. Make sure to use the DTrace flag while compiling. Recompiling will
+create the appropriate files using the system installation of DTrace.
+
+### Extending a probe
+
+We can extend a probe by adding an extra argument to the probe. Like adding a
+probe, the probe definition needs to be appended in
+[`src/common/dtrace_probes.d`](../../src/common/dtrace_probes.d). Note that this
+extra information needs to be available **everywhere** the probe is used.
+
+Once you've added the argument, you need to change **all** instances where the
+probe is in use. Note that you have to both add the new argument and change the
+`DTRACE` macro. Then you can recompile the PonyC compiler to use your new
+arguments.
+
+*Do not forget that if you change the order of the arguments for any of the
+existing nodes, you also need to change their usage in the example scripts.*

examples/dtrace/gc.d

@@ -0,0 +1,22 @@
+#!/usr/bin/env dtrace -s
+
+#pragma D option quiet
+
+pony$target:::gc-start
+{
+  @count["GC Passes"] = count();
+  self->start_gc = timestamp;
+}
+
+pony$target:::gc-end
+{
+  @quant["Time in GC (ns)"] = quantize(timestamp - self->start_gc);
+  @times["Total time"] = sum(timestamp - self->start_gc);
+}
+
+END
+{
+  printa(@count);
+  printa(@quant);
+  printa(@times);
+}

examples/dtrace/telemetry.d

@@ -0,0 +1,103 @@
+#!/usr/bin/env dtrace -x aggsortkey -x aggsortkeypos=1 -s
+
+#pragma D option quiet
+
+inline unsigned int UINT32_MAX = -1;
+inline unsigned int ACTORMSG_BLOCK = (UINT32_MAX - 6);
+inline unsigned int ACTORMSG_UNBLOCK = (UINT32_MAX - 5);
+inline unsigned int ACTORMSG_ACQUIRE = (UINT32_MAX - 4);
+inline unsigned int ACTORMSG_RELEASE = (UINT32_MAX - 3);
+inline unsigned int ACTORMSG_CONF = (UINT32_MAX - 2);
+inline unsigned int ACTORMSG_ACK = (UINT32_MAX - 1);
+
+pony$target:::actor-msg-send
+/ (unsigned int)arg1 == (unsigned int)ACTORMSG_BLOCK /
+{
+  @counts[arg0, "Block Messages Sent"] = count();
+}
+
+pony$target:::actor-msg-send
+/ (unsigned int)arg1 == (unsigned int)ACTORMSG_UNBLOCK /
+{
+  @counts[arg0, "Unblock Messages Sent"] = count();
+}
+
+pony$target:::actor-msg-send
+/ (unsigned int)arg1 == (unsigned int)ACTORMSG_ACQUIRE /
+{
+  @counts[arg0, "Acquire Messages Sent"] = count();
+}
+
+pony$target:::actor-msg-send
+/ (unsigned int)arg1 == (unsigned int)ACTORMSG_RELEASE /
+{
+  @counts[arg0, "Release Messages Sent"] = count();
+}
+
+pony$target:::actor-msg-send
+/ (unsigned int)arg1 == (unsigned int)ACTORMSG_CONF /
+{
+  @counts[arg0, "Confirmation Messages Sent"] = count();
+}
+
+pony$target:::actor-msg-send
+/ (unsigned int)arg1 == (unsigned int)ACTORMSG_ACK /
+{
+  @counts[arg0, "Acknowledgement Messages Sent"] = count();
+}
+
+pony$target:::actor-msg-send
+/ (unsigned int)arg1 < (unsigned int)ACTORMSG_BLOCK /
+{
+  @counts[arg0, "Application Messages Sent"] = count();
+}
+
+pony$target:::actor-alloc
+{
+  @counts[arg0, "Actor Allocations"] = count();
+}
+
+pony$target:::heap-alloc
+{
+  @counts[arg0, "Heap Allocations"] = count();
+  @sizes[arg0, "Heap Allocations Size"] = sum(arg1);
+}
+
+pony$target:::gc-start
+{
+  @counts[arg0, "GC Passes"] = count();
+  self->start_gc = timestamp;
+}
+
+pony$target:::gc-end
+{
+  @times[arg0, "Time in GC"] = sum(timestamp - self->start_gc);
+}
+
+pony$target:::gc-send-start
+{
+  @counts[arg0, "Objects Sent"] = count();
+  self->start_send = timestamp;
+}
+
+pony$target:::gc-send-end
+{
+  @times[arg0, "Time in Send Scan"] = sum(timestamp - self->start_send);
+}
+
+pony$target:::gc-recv-start
+{
+  @counts[arg0, "Objects Received"] = count();
+  self->start_recv = timestamp;
+}
+
+pony$target:::gc-recv-end
+{
+  @times[arg0, "Time in Recv Scan"] = sum(timestamp - self->start_recv);
+}
+
+END
+{
+  printf("%?s  %-40s %10s %10s %10s\n", "SCHEDULER", "EVENT", "COUNT", "TIME (ns)", "SIZE");
+  printa("%?p  %-40s %@10d %@10d %@10d\n", @counts, @times, @sizes);
+}