Enhance some of GC features

* Add a heuristic to run GC before heap block allocation
  (to reduce fragmentation overhead)
* Add logger & visualizer of runtime heap usage
  (available both in debug/release mode)

.gitignore

@@ -56,3 +56,4 @@ android/obj
 android/libs
 #test/chakracore/chakracorelog.verbose.txt
 *.gen.txt
+bdwgc_log

build/Flags.mk

@@ -13,6 +13,9 @@ ESCARGOT_CXXFLAGS_COMMON += -Wno-unused-but-set-variable -Wno-unused-but-set-par
 ESCARGOT_CXXFLAGS_COMMON += -Wno-type-limits -Wno-unused-result -Wno-unused-variable # TODO: enable these warnings
 ESCARGOT_CXXFLAGS_COMMON += -Wno-deprecated-declarations
 
+#ESCARGOT_CXXFLAGS_COMMON += -DPROFILE_MASSIF
+#ESCARGOT_CXXFLAGS_COMMON += -DPROFILE_BDWGC
+
 ESCARGOT_LDFLAGS_COMMON += -lpthread
 ESCARGOT_LDFLAGS_COMMON += -lrt
 

build_third_party.sh

@@ -27,6 +27,9 @@ automake --add-missing
 
 GCCONFFLAGS_COMMON=" --disable-parallel-mark " # --enable-large-config --enable-cplusplus"
 CFLAGS_COMMON=" -g3 "
+CFLAGS_COMMON+=" -DESCARGOT "
+CFLAGS_COMMON+=" -fdata-sections -ffunction-sections " # To exclude unused code from final binary
+CFLAGS_COMMON+=" -DIGNORE_DYNAMIC_LOADING -DGC_DONT_REGISTER_MAIN_STATIC_DATA " # Everything in global data is false reference
 LDFLAGS_COMMON=
 
 # HOST flags : linux / wearable / mobile / tv -------------

src/shell/shell.cpp

@@ -112,6 +112,11 @@ void GC_free_hook(void* address)
 
 #endif
 
+#ifdef PROFILE_BDWGC
+const char* bdwgc_log_filename = "bdwgc_log";
+const char* bdwgc_log_phase = "initial phase";
+#endif
+
 void eval(Escargot::Context* context, Escargot::String* str, Escargot::String* fileName, bool shouldPrintScriptResult)
 {
     auto result = context->scriptParser().parse(str, fileName);
@@ -173,6 +178,19 @@ int main(int argc, char* argv[])
         }
     });
 #endif
+#ifdef PROFILE_BDWGC
+    remove(bdwgc_log_filename);
+    FILE* fp = fopen(bdwgc_log_filename, "a");
+    if (fp) {
+        fprintf(fp, "GC_no    PeakRSS   TotalHeap    Marked  # Phase\n");
+        fclose(fp);
+    }
+    GC_set_on_collection_event([](GC_EventType evtType) {
+        if (GC_EVENT_RECLAIM_START == evtType) {
+            GC_dump_for_graph(bdwgc_log_filename, bdwgc_log_phase);
+        }
+    });
+#endif
 #ifndef NDEBUG
     setbuf(stdout, NULL);
     setbuf(stderr, NULL);

third_party/bdwgc/allchblk.c

@@ -582,6 +582,17 @@ GC_allochblk(size_t sz, int kind, unsigned flags/* IGNORE_OFF_PAGE or 0 */)
     int split_limit; /* Highest index of free list whose blocks we      */
                      /* split.                                          */
 
+#ifdef ESCARGOT
+    if (GC_get_bytes_since_gc() > 10 * 1024 * 1024) {
+        /*
+         * To reduce fragmentation overhead,
+         * collect occasionally before allocating new block
+         * if many objects have been allocated without GC.
+         */
+        GC_gcollect_inner();
+    }
+#endif
+
     GC_ASSERT((sz & (GRANULE_BYTES - 1)) == 0);
     blocks = OBJ_SZ_TO_BLOCKS(sz);
     if ((signed_word)(blocks * HBLKSIZE) < 0) {

third_party/bdwgc/dbg_mlc.c

@@ -187,6 +187,9 @@
     GC_ref_kind source;
     size_t offset;
     void *base;
+#ifdef ESCARGOT
+    ptr_t object_start;
+#endif
 
     GC_print_heap_obj(GC_base(current));
 
@@ -203,7 +206,12 @@
       GC_err_printf("Reachable via %d levels of pointers from ", i);
       switch(source) {
         case GC_REFD_FROM_ROOT:
+#ifndef ESCARGOT
           GC_err_printf("root at %p\n\n", base);
+#else
+          /* Print more detailed information in backtrace */
+          GC_err_printf("root at %p (=> points %p)\n\n", base, *((void**)base));
+#endif
           goto out;
         case GC_REFD_FROM_REG:
           GC_err_printf("root in register\n\n");
@@ -212,7 +220,18 @@
           GC_err_printf("list of finalizable objects\n\n");
           goto out;
         case GC_REFD_FROM_HEAP:
+#ifndef ESCARGOT
           GC_err_printf("offset %ld in object:\n", (long)offset);
+#else
+          /* Print more detailed information in backtrace */
+          object_start = GC_base(base) + sizeof(oh);
+          GC_bool interior = ((*((void**)(object_start + offset))) != current);
+          GC_err_printf("offset %ld in object %p (=> points %p%s):\n",
+                        (long)offset,
+                        object_start,
+                        *((void**)(object_start + offset)),
+                        interior?", interior":"");
+#endif
           /* Take GC_base(base) to get real base, i.e. header. */
           GC_print_heap_obj(GC_base(base));
           break;

third_party/bdwgc/include/gc.h

@@ -1504,6 +1504,13 @@ GC_API void * GC_CALL GC_is_valid_displacement(void * /* p */);
 /* Defined only if the library has been compiled without NO_DEBUGGING.  */
 GC_API void GC_CALL GC_dump(void);
 
+#ifdef ESCARGOT
+/* This function appends the GC status log to give file                 */
+/* which can be interpreted by gnuplot (and MS Excel of course).        */
+GC_API void GC_CALL GC_dump_for_graph(const char* /* log_file_name */,
+                                      const char* /* phase_name */);
+#endif
+
 /* Safer, but slow, pointer addition.  Probably useful mainly with      */
 /* a preprocessor.  Useful only for heap pointers.                      */
 /* Only the macros without trailing digits are meant to be used         */

third_party/bdwgc/misc.c

@@ -1310,6 +1310,10 @@ GC_API void GC_CALL GC_init(void)
         GC_init_dyld();
 #   endif
     RESTORE_CANCEL(cancel_state);
+#ifdef ESCARGOT
+    GC_heapsize = 0;
+    GC_unmapped_bytes = 0;
+#endif
 }
 
 GC_API void GC_CALL GC_enable_incremental(void)

third_party/bdwgc/reclaim.c

@@ -581,6 +581,141 @@ void GC_print_free_list(int kind, size_t sz_in_granules)
 
 #endif /* !NO_DEBUGGING */
 
+#ifdef ESCARGOT
+
+/*
+ * Below code will always be compiled (i.e., both in debug/release mode).
+ *
+ * But it won't be included in final binary if GC_dump_for_graph doesn't get called in escargot.
+ * So adding '-fdata-sections -ffunction-sections' into CFLAG is necessary when building bdwgc
+ */
+
+#include <sys/resource.h>
+
+#if defined(NO_DEBUGGING)
+
+#ifdef USE_MARK_BYTES
+
+int GC_n_set_marks(hdr *hhdr)
+{
+    int result = 0;
+    int i;
+    size_t sz = hhdr -> hb_sz;
+    int offset = (int)MARK_BIT_OFFSET(sz);
+    int limit = (int)FINAL_MARK_BIT(sz);
+
+    for (i = 0; i < limit; i += offset) {
+        result += hhdr -> hb_marks[i];
+    }
+    GC_ASSERT(hhdr -> hb_marks[limit]);
+    return(result);
+}
+
+#else // USE_MARK_BYTES
+
+static int set_bits(word n)
+{
+    word m = n;
+    int result = 0;
+
+    while (m > 0) {
+        if (m & 1) result++;
+        m >>= 1;
+    }
+    return(result);
+}
+
+int GC_n_set_marks(hdr *hhdr)
+{
+    int result = 0;
+    int i;
+    int n_mark_words;
+#   ifdef MARK_BIT_PER_OBJ
+      int n_objs = (int)HBLK_OBJS(hhdr -> hb_sz);
+
+      if (0 == n_objs) n_objs = 1;
+      n_mark_words = divWORDSZ(n_objs + WORDSZ - 1);
+#   else /* MARK_BIT_PER_GRANULE */
+      n_mark_words = MARK_BITS_SZ;
+#   endif
+    for (i = 0; i < n_mark_words - 1; i++) {
+        result += set_bits(hhdr -> hb_marks[i]);
+    }
+#   ifdef MARK_BIT_PER_OBJ
+      result += set_bits((hhdr -> hb_marks[n_mark_words - 1])
+                         << (n_mark_words * WORDSZ - n_objs));
+#   else
+      result += set_bits(hhdr -> hb_marks[n_mark_words - 1]);
+#   endif
+    return(result - 1);
+}
+
+#endif // USE_MARK_BYTES
+
+#endif // defined(NO_DEBUGGING)
+
+struct Print_stats_escargot
+{
+        size_t number_of_blocks;
+        size_t total_bytes;
+        size_t marked_num;
+        size_t marked_bytes;
+};
+
+STATIC void GC_gather_information_for_escargot(struct hblk *h,
+                                               word raw_pse)
+{
+    hdr * hhdr = HDR(h);
+    size_t bytes = hhdr -> hb_sz;
+    struct Print_stats_escargot *pse;
+    unsigned n_marks = GC_n_set_marks(hhdr);
+
+    GC_ASSERT(hhdr -> hb_n_marks == n_marks);
+
+    bytes += HBLKSIZE-1;
+    bytes &= ~(HBLKSIZE-1);
+
+    pse = (struct Print_stats_escargot *)raw_pse;
+    pse->number_of_blocks++;
+    pse->total_bytes += bytes;
+    pse->marked_num += n_marks;
+    pse->marked_bytes += (hhdr->hb_sz) * n_marks;
+}
+
+GC_API void GC_CALL GC_dump_for_graph(const char* log_file_name,
+                                      const char* phase_name)
+{
+    struct Print_stats_escargot pstats;
+
+    pstats.number_of_blocks = 0;
+    pstats.total_bytes = 0;
+    pstats.marked_num = 0;
+    pstats.marked_bytes = 0;
+
+    GC_apply_to_all_blocks(GC_gather_information_for_escargot, (word)&pstats);
+
+    struct rusage ru;
+    getrusage(RUSAGE_SELF, &ru);
+    size_t peak_rss = ru.ru_maxrss;
+
+    GC_printf("[%d] %s : PeakRSS %zu KB, TotalHeap %lu KB, MarkedHeap %lu KB\n",
+              GC_get_gc_no(), phase_name, peak_rss,
+              (unsigned long) pstats.total_bytes / 1024,
+              (unsigned long) pstats.marked_bytes / 1024);
+
+    FILE* fp = fopen(log_file_name, "a");
+    if (fp) {
+        fprintf(fp, "%5d %9lu %9lu %9zu     # %s\n",
+                GC_get_gc_no(),
+                peak_rss,
+                (unsigned long) pstats.total_bytes / 1024,
+                (unsigned long) pstats.marked_bytes / 1024,
+                phase_name);
+        fclose(fp);
+    }
+}
+#endif // ESCARGOT
+
 /*
  * Clear all obj_link pointers in the list of free objects *flp.
  * Clear *flp.

tools/draw_bdwgc_plot.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+
+# Copyright 2015 Samsung Electronics Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import os
+
+BDWGC_LOGFILE="bdwgc_log"
+GNUPLOT_DISPLAY_STYLE="lines"
+
+def make_plot_subcmd(col, name):
+    return '\"%s\" using 1:%d title \"%s (KB)\" with %s' % (BDWGC_LOGFILE, col, name, GNUPLOT_DISPLAY_STYLE)
+
+def draw_bdwgc_plot():
+    if not os.path.exists(BDWGC_LOGFILE):
+        print 'Cannot draw plot! No input file %s' % BDWGC_LOGFILE
+        print 'Re-build escargot with `-DPROFILE_BDWGC`'
+        exit(1)
+
+    plot_cmd = 'plot '
+    plot_cmd += make_plot_subcmd(2, "Peak RSS")
+    plot_cmd += ', '
+    plot_cmd += make_plot_subcmd(3, "Total Heap")
+    plot_cmd += ', '
+    plot_cmd += make_plot_subcmd(4, "Marked Heap")
+
+    print "gnuplot -p -e '%s'" % plot_cmd
+    print
+    print "See '%s' to see raw data." % BDWGC_LOGFILE
+
+    subprocess.call(['gnuplot', '-p', '-e', plot_cmd])
+
+if __name__ == '__main__':
+    draw_bdwgc_plot()