Merge pull request #34 from nctl144/update_source_new

Update chromium source

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "third_party/chromium/url"]
+	path = third_party/chromium/url
+	url = https://github.com/nctl144/url-chromium
+[submodule "third_party/chromium/base"]
+	path = third_party/chromium/base
+	url = https://github.com/nctl144/base-chromium

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-sudo: false
+sudo: required
 branches:
   only:
     - master
@@ -19,24 +19,37 @@ matrix:
       env: TOXENV=pypy3
     - python: 3.6
       env: TOXENV=py36
+dist: trusty
 install:
-    - |
-      if [ "$TOXENV" = "pypy" ]; then
-        export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
-        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
-        tar -jxf ${PYPY_VERSION}.tar.bz2
-        virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
-        source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-      fi
-      if [ "$TOXENV" = "pypy3" ]; then
-        export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
-        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
-        tar -jxf ${PYPY_VERSION}.tar.bz2
-        virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
-        source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-      fi
-    - pip install -U pip tox Cython codecov
+  - |
+    if [ "$TOXENV" = "pypy" ]; then
+      export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
+      wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
+      tar -jxf ${PYPY_VERSION}.tar.bz2
+      virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
+      source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+    fi
+    if [ "$TOXENV" = "pypy3" ]; then
+      export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
+      wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
+      tar -jxf ${PYPY_VERSION}.tar.bz2
+      virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
+      source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+    fi
+  - pip install -U pip tox Cython codecov
+addons:
+  apt:
+    sources:
+    - ubuntu-toolchain-r-test
+    packages:
+    - gcc-5
+    - g++-5
+before_install: sudo apt-get update -qq
 script:
-    - tox
+  - sudo unlink /usr/bin/gcc && sudo ln -s /usr/bin/gcc-5 /usr/bin/gcc
+  - sudo unlink /usr/bin/g++ && sudo ln -s /usr/bin/g++-5 /usr/bin/g++
+  - gcc --version
+  - g++ --version
+  - tox
 after_success:
-    - codecov
+  - codecov

MANIFEST.in

@@ -1,4 +1,4 @@
-recursive-include vendor *.cpp *.h *.cc
+recursive-include third_party *.cpp *.h *.cc
 recursive-include scurl *.pxd *.pyx *.cpp
 include README.md
 include LICENSE

scurl/chromium_gurl.pxd

@@ -3,7 +3,7 @@ from libcpp cimport bool
 from mozilla_url_parse cimport Component, Parsed
 
 
-cdef extern from "../vendor/gurl/url/gurl.h":
+cdef extern from "../third_party/chromium/url/gurl.h":
     cdef cppclass GURL:
         GURL()
         GURL(const string & url_string)

scurl/chromium_url_canon.pxd

@@ -3,7 +3,7 @@ from mozilla_url_parse cimport Component, Parsed
 from chromium_url_canon_stdstring cimport StdStringCanonOutput
 
 
-cdef extern from "../vendor/gurl/url/url_canon.h" namespace "url":
+cdef extern from "../third_party/chromium/url/url_canon.h" namespace "url":
     cdef cppclass CanonOutputT:
         pass
 

scurl/chromium_url_canon_stdstring.pxd

@@ -1,7 +1,7 @@
 from libcpp.string cimport string
 
 
-cdef extern from "../vendor/gurl/url/url_canon_stdstring.h" namespace "url":
+cdef extern from "../third_party/chromium/url/url_canon_stdstring.h" namespace "url":
     cdef cppclass StdStringCanonOutput:
         StdStringCanonOutput()
         StdStringCanonOutput(string* str)

scurl/chromium_url_constant.pxd

@@ -5,7 +5,7 @@ from libcpp cimport bool
 from mozilla_url_parse cimport Component, Parsed
 
 
-cdef extern from "../vendor/gurl/url/url_constants.h" namespace "url":
+cdef extern from "../third_party/chromium/url/url_constants.h" namespace "url":
 
     extern const char kAboutBlankURL[];
 

scurl/chromium_url_util.pxd

@@ -5,7 +5,7 @@ from chromium_url_canon cimport CharsetConverter
 from chromium_url_canon_stdstring cimport StdStringCanonOutput
 
 
-cdef extern from "../vendor/gurl/url/url_util.h" namespace "url":
+cdef extern from "../third_party/chromium/url/url_util.h" namespace "url":
     cdef bool IsStandard(const char* spec, const Component& scheme)
     cdef bool Canonicalize(const char* spec,
                            int spec_len,

scurl/chromium_url_util_internal.pxd

@@ -3,7 +3,7 @@ from libcpp cimport bool
 from mozilla_url_parse cimport Component, Parsed
 
 
-cdef extern from "../vendor/gurl/url/url_util_internal.h" namespace "url":
+cdef extern from "../third_party/chromium/url/url_util_internal.h" namespace "url":
     cdef bool CompareSchemeComponent(const char* spec,
                                      const Component& component,
                                      const char* compare_to)

scurl/mozilla_url_parse.pxd

@@ -1,7 +1,7 @@
 from libcpp cimport bool
 
 
-cdef extern from "../vendor/gurl/url/third_party/mozilla/url_parse.h" namespace "url":
+cdef extern from "../third_party/chromium/url/third_party/mozilla/url_parse.h" namespace "url":
     cdef struct Component:
         int begin
         int len

setup.py

@@ -4,6 +4,7 @@
 from os.path import splitext
 import logging
 import platform
+from glob import glob
 
 VERSION = "0.1.0"
 ext_macros = []
@@ -24,49 +25,58 @@
         logger.warning('Warning: Enabling line tracing in Cython extension.\
                         This will make the performance of the library less effective!')
 
+ext_sources = [
+    'scurl/cgurl.pyx',
+    'third_party/chromium/base/callback_internal.cc',
+    'third_party/chromium/base/at_exit.cc',
+    'third_party/chromium/base/lazy_instance_helpers.cc',
+    'third_party/chromium/base/strings/utf_string_conversion_utils.cc',
+    'third_party/chromium/base/strings/string_piece.cc',
+    'third_party/chromium/base/strings/string16.cc',
+    'third_party/chromium/base/strings/string_util.cc',
+    'third_party/chromium/base/strings/utf_string_conversions.cc',
+    'third_party/chromium/base/strings/string_util_constants.cc',
+    'third_party/chromium/base/third_party/icu/icu_utf.cc',
+    'third_party/chromium/url/gurl.cc',
+    'third_party/chromium/url/url_canon.cc',
+    'third_party/chromium/url/url_canon_etc.cc',
+    # 'third_party/chromium/url/url_canon_icu.cc',
+    'third_party/chromium/url/url_canon_filesystemurl.cc',
+    'third_party/chromium/url/url_canon_fileurl.cc',
+    'third_party/chromium/url/url_canon_host.cc',
+    'third_party/chromium/url/url_canon_internal.cc',
+    'third_party/chromium/url/url_canon_ip.cc',
+    'third_party/chromium/url/url_canon_mailtourl.cc',
+    'third_party/chromium/url/url_canon_path.cc',
+    'third_party/chromium/url/url_canon_pathurl.cc',
+    'third_party/chromium/url/url_canon_query.cc',
+    'third_party/chromium/url/url_canon_relative.cc',
+    'third_party/chromium/url/url_canon_stdstring.cc',
+    'third_party/chromium/url/url_canon_stdurl.cc',
+    'third_party/chromium/url/url_constants.cc',
+    'third_party/chromium/url/url_parse_file.cc',
+    'third_party/chromium/url/url_util.cc',
+    'third_party/chromium/url/third_party/mozilla/url_parse.cc',
+]
+
 extension = [
     Extension(
         name="scurl.cgurl",
-        sources=["scurl/cgurl.pyx",
-                 "vendor/gurl/base/third_party/icu/icu_utf.cc",
-                 "vendor/gurl/base/strings/string16.cc",
-                 "vendor/gurl/base/strings/string_piece.cc",
-                 "vendor/gurl/base/strings/string_util.cc",
-                 "vendor/gurl/base/strings/utf_string_conversions.cc",
-                 "vendor/gurl/base/strings/utf_string_conversion_utils.cc",
-                 "vendor/gurl/url/gurl.cc",
-                 "vendor/gurl/url/url_canon_etc.cc",
-                 "vendor/gurl/url/url_canon_filesystemurl.cc",
-                 "vendor/gurl/url/url_canon_fileurl.cc",
-                 "vendor/gurl/url/url_canon_host.cc",
-                 "vendor/gurl/url/url_canon_internal.cc",
-                 "vendor/gurl/url/url_canon_ip.cc",
-                 "vendor/gurl/url/url_canon_mailtourl.cc",
-                 "vendor/gurl/url/url_canon_path.cc",
-                 "vendor/gurl/url/url_canon_pathurl.cc",
-                 "vendor/gurl/url/url_canon_query.cc",
-                 "vendor/gurl/url/url_canon_relative.cc",
-                 "vendor/gurl/url/url_canon_stdstring.cc",
-                 "vendor/gurl/url/url_canon_stdurl.cc",
-                 "vendor/gurl/url/url_constants.cc",
-                 "vendor/gurl/url/url_parse_file.cc",
-                 "vendor/gurl/url/url_util.cc",
-                 "vendor/gurl/url/third_party/mozilla/url_parse.cc"
-                 ],
+        sources=ext_sources,
         language="c++",
-        extra_compile_args=["-std=gnu++0x", "-I./vendor/gurl/",
-                            "-fPIC", "-Ofast", "-pthread", "-w"],
-        extra_link_args=["-std=gnu++0x", "-w"],
+        extra_compile_args=["-std=gnu++14", "-I./third_party/chromium/",
+                            "-fPIC", "-Ofast", "-pthread", "-w", '-DU_COMMON_IMPLEMENTATION'],
+        extra_link_args=["-std=gnu++14", "-w"],
         include_dirs=['.'],
         define_macros=ext_macros
     ),
     Extension(
         name="scurl.canonicalize",
         sources=["scurl/canonicalize.pyx"],
         language="c++",
-        extra_compile_args=["-std=gnu++0x", "-I./vendor/gurl/",
+        extra_compile_args=["-std=gnu++14", "-I./third_party/chromium/",
                             "-fPIC", "-Ofast", "-pthread", "-w"],
-        extra_link_args=["-std=gnu++0x", "-w"],
+        extra_link_args=["-std=gnu++14", "-w"],
         include_dirs=['.'],
         define_macros=ext_macros
     )

third_party/chromium/README.md

@@ -0,0 +1,16 @@
+## Notes on updating the chromium source for SCURL project
+
+Updating the source for the project might be a lot of work. But sometimes there might be a lot of functionality
+implemented in the chromium source code that we might miss if we don't update it regularly
+
+Here are the forked repository of the base component and the url component:
+
++ [base-chromium](https://github.com/nctl144/base-chromium)
++ [url-chromium](https://github.com/nctl144/url-chromium)
+
+Since base component is really large and we might not even need all of the components defined in it, there are a
+few functions/files that we need to skip:
+
++ YieldCurrentThread
++ lock
++ platforms-related components