track libcurl and libuv with cmake, add libuv submodule

.gitlab-ci.yml

@@ -6,20 +6,6 @@ stages:
   - build
   - test
 
-# disabled because gcc9 can't build net_if.hpp
-#build:linux:
-#  image: alpine:latest
-#  tags:
-#    - linux
-#  stage: build
-#  before_script:
-#    - apk add --update g++ make cmake linux-headers libcap-dev libuv-dev curl-dev git
-#  script:
-#    - make STATIC_LINK=ON DOWNLOAD_SODIUM=ON
-#  artifacts:
-#    paths:
-#      - "lokinet"
-
 # we'll just try our travis set up for now
 build:linux_release:
   image: ubuntu:xenial
@@ -33,7 +19,7 @@ build:linux_release:
     - apt-get update && apt-get install -y build-essential ca-certificates cmake curl git libcap-dev pkg-config wget xz-utils python3-pip
     - pip3 install git-archive-all
   script:
-    - make static DOWNLOAD_SODIUM=ON DOWNLOAD_UV=ON
+    - mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DDOWNLOAD_CURL=ON -DDOWNLOAD_SODIUM=ON -DSTATIC_LINK_RUNTIME=ON && make
     # alternatively: git describe --tag
     - LOKINET_VERSION=$(./lokinet-static --version|sed s/lokinet-//)
     - echo lokinet-linux-x64-v$LOKINET_VERSION
@@ -58,7 +44,7 @@ build:linux_debug:
     - echo 'deb https://apt.kitware.com/ubuntu/ xenial main' >/etc/apt/sources.list.d/kitware.list
     - apt-get update && apt-get install -y build-essential ca-certificates cmake curl git libcap-dev libcurl4-openssl-dev libuv1-dev ninja-build
   script:
-    - DOWNLOAD_SODIUM=ON IS_NOTIFICATION=1 STATIC_LINK=OFF make
+    - mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Debug -DDOWNLOAD_CURL=ON -DDOWNLOAD_SODIUM=ON -DSTATIC_LINK_RUNTIME=ON && make package
   artifacts:
     paths:
       - build/daemon/lokinet
@@ -73,7 +59,7 @@ build:win32:
   before_script:
     - apt-get update && apt-get install -y build-essential cmake nsis git g++-mingw-w64
   script:
-    - DOWNLOAD_SODIUM=ON IS_NOTIFICATION=1 STATIC_LINK=ON WINDOWS_ARCH=64 make windows
+    - mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=../contrib/cross/mingw64.cmake -DSTATIC_LINK_RUNTIME=ON && make package
   artifacts:
     paths:
       - build/lokinet-*.exe
@@ -88,29 +74,3 @@ build:macos:
     paths:
       - build/*.pkg
       - build/daemon/lokinet
-
-
-# needs libuv and libcurl
-#build:freebsd:
-#  tags:
-#     - freebsd
-#  stage: build
-#  script:
-#    - gmake DOWNLOAD_SODIUM=ON
-#  artifacts:
-#    paths:
-#      - "lokinet"
-
-# need windows runner on lokiproject
-#build:windows:
-#  tags:
-#    - windows
-#  stage: build
-#  script:
-    #- make static CC=gcc CXX=g++
-    # can't use STATIC because LTO doesn't work on cross-compiled targets
-    # -DSTATIC_LINK_RUNTIME=ON -DLIBUV_ROOT=libuv
-#    - cmake .. -G Ninja -DCMAKE_TOOLCHAIN_FILE=../contrib/cross/mingw[32].cmake  -DDOWNLOAD_SODIUM=ON
-#  artifacts:
-#    paths:
-#      - "lokinet.exe"

.gitmodules

@@ -20,3 +20,7 @@
 [submodule "external/clang-format-hooks"]
 	path = external/clang-format-hooks
 	url = https://github.com/barisione/clang-format-hooks/
+[submodule "external/libuv"]
+	path = external/libuv
+	url = https://github.com/libuv/libuv
+	branch = v1.x

.travis.yml

@@ -9,13 +9,15 @@ matrix:
   - name: "lint check"
     os: linux
     compiler: gcc
-    env: MAKE_TARGET=format-verify PATH="/usr/lib/llvm-9/bin:$PATH"
+    env: PATH="/usr/lib/llvm-9/bin:$PATH"
     addons:
       apt:
         sources:
           - llvm-toolchain-bionic-9
         packages:
           - clang-format-9
+    script:
+      - make format-verify
   - name: "make debug (linux/gcc)"
     os: linux
     compiler: gcc
@@ -209,11 +211,11 @@ before_install:
 
 script:
   - if [ "$TRAVIS_OS_NAME" == "windows" ]; then
-      cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && make test';
+      cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 mkdir build && cd build && cmake .. && make test';
     elif [[ ! -z $DOCKER_FILE ]]; then
       docker build -f $DOCKER_FILE .;
     else
-      make DOWNLOAD_SODIUM=ON ${MAKE_TARGET:-test};
+      mkdir build && cd build && cmake .. && make DOWNLOAD_SODIUM=ON ${MAKE_TARGET:-test};
     fi
 
 after_script:

CMakeLists.txt

@@ -287,6 +287,7 @@ if(SUBMODULE_CHECK)
     check_submodule(external/cxxopts)
     check_submodule(external/date)
     check_submodule(external/pybind11)
+    check_submodule(external/libuv)
   endif()
 endif()
 

Makefile

@@ -1,5 +1,3 @@
-
-
 REPO := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
 

cmake/DownloadLibCurl.cmake

@@ -0,0 +1,46 @@
+set(LIBCURL_PREFIX ${CMAKE_BINARY_DIR}/libcurl)
+set(LIBCURL_URL https://github.com/curl/curl/releases/download/curl-7_67_0/curl-7.67.0.tar.xz)
+set(LIBCURL_HASH SHA256=f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c)
+
+if(CURL_TARBALL_URL)
+    # make a build time override of the tarball url so we can fetch it if the original link goes away
+    set(LIBCURL_URL ${CURL_TARBALL_URL})
+endif()
+
+
+file(MAKE_DIRECTORY ${LIBCURL_PREFIX}/include)
+
+include(ExternalProject)
+include(ProcessorCount)
+
+set(CURL_CONFIGURE_FLAGS_COMMON --prefix=${LIBCURL_PREFIX} --without-ssl --without-nss --without-ca-bundle --without-libidn2 --without-zlib --without-librtmp --without-ca-fallback --without-ca-path --enable-static --disable-shared CC=${CMAKE_C_COMPILER})
+
+if (CMAKE_C_COMPILER_ARG1)
+  set(CURL_CONFIGURE_FLAGS_COMMON ${CURL_CONFIGURE_FLAGS_COMMON} CPPFLAGS=${CMAKE_C_COMPILER_ARG1})
+endif()
+
+set(CURL_CONFIGURE ./configure ${CURL_CONFIGURE_FLAGS_COMMON})
+
+
+if (CROSS_TARGET)
+    set(CURL_CONFIGURE ${CURL_CONFIGURE} --target=${CROSS_TARGET} --host=${CROSS_TARGET})
+endif()
+
+
+ExternalProject_Add(libcurl_external
+    BUILD_IN_SOURCE ON
+    PREFIX ${LIBCURL_PREFIX}
+    URL ${LIBCURL_URL}
+    URL_HASH ${LIBCURL_HASH}
+    CONFIGURE_COMMAND ${CURL_CONFIGURE}
+    BUILD_COMMAND make -j${PROCESSOR_COUNT}
+    INSTALL_COMMAND ${MAKE}
+    BUILD_BYPRODUCTS ${LIBCURL_PREFIX}/lib/libcurl.a ${LIBCURL_PREFIX}/include
+)
+
+add_library(curl_vendor STATIC IMPORTED GLOBAL)
+add_dependencies(curl_vendor curl_external)
+set_target_properties(curl_vendor PROPERTIES
+    IMPORTED_LOCATION ${LIBCURL_PREFIX}/lib/libcurl.a
+    INTERFACE_INCLUDE_DIRECTORIES ${LIBCURL_PREFIX}/include
+)

cmake/DownloadLibSodium.cmake

@@ -19,8 +19,8 @@ else()
   set(SODIUM_CONFIGURE ./configure --prefix=${LIBSODIUM_PREFIX} --enable-static --disable-shared CC=${CMAKE_C_COMPILER})
 endif()
 
-if (LIBSODIUM_CROSS_TARGET)
-    set(SODIUM_CONFIGURE ${SODIUM_CONFIGURE} --target=${LIBSODIUM_CROSS_TARGET} --host=${LIBSODIUM_CROSS_TARGET})
+if (CROSS_TARGET)
+    set(SODIUM_CONFIGURE ${SODIUM_CONFIGURE} --target=${CROSS_TARGET} --host=${CROSS_TARGET})
 endif()
 
 

cmake/unix.cmake

@@ -2,25 +2,24 @@ if(NOT UNIX)
   return()
 endif()
 
-# can't have UNIX defined
-# can't have the include paths for libuv either?
-if(NOT IOS AND NOT ANDROID AND NOT WIN32)
-  if(LIBCURL_ROOT)
-    option(HTTP_ONLY "" ON)
-    option(SSL_ENABLED "" OFF)
-    option(BUILD_SHARED_LIBS "" OFF)
-    option(ENABLE_IPV6 "" OFF)
-    option(CMAKE_USE_OPENSSL "" OFF)
-    option(CURL_CA_PATH "" OFF)
-    set(CURL_CA_PATH none)
-    message(STATUS "using local curl: ${LIBCURL_ROOT}")
-    add_subdirectory(${LIBCURL_ROOT})
-    set(CURL_INCLUDE_DIRS ${LIBCURL_ROOT}/include)
-    set(CURL_LIBRARIES libcurl)
-    set(CURL_FOUND TRUE)
-  else()
-    include(FindCURL)
-  endif()
+include(CheckCXXSourceCompiles)
+include(CheckLibraryExists)
+
+
+option(DOWNLOAD_CURL "download and statically compile in CURL" OFF)
+# Allow -DDOWNLOAD_CURL=FORCE to download without even checking for a local libcurl
+if(NOT DOWNLOAD_CURL STREQUAL "FORCE")
+  include(FindCURL)
+endif()
+
+if(CURL_FOUND)
+  message(STATUS "using system curl")
+elseif(DOWNLOAD_CURL)
+  message(STATUS "libcurl not found, but DOWNLOAD_CURL specified, so downloading it")
+  include(DownloadLibCurl)
+  set(CURL_LIBRARIES curl_vendor)
+else()
+  message(FATAL_ERROR "Could not find libcurl; either install it on your system or use -DDOWNLOAD_CURL=ON to download and build an internal copy")
 endif()
 
 add_definitions(-DUNIX)
@@ -31,14 +30,22 @@ if (STATIC_LINK_RUNTIME OR STATIC_LINK)
   set(LIBUV_USE_STATIC ON)
 endif()
 
-if(LIBUV_ROOT)
+
+option(DOWNLOAD_UV "statically compile in libuv" OFF)
+# Allow -DDOWNLOAD_UV=FORCE to download without even checking for a local libuv
+if(NOT DOWNLOAD_UV STREQUAL "FORCE")
+  find_package(LibUV 1.28.0)
+endif()
+if(LibUV_FOUND)
+  message(STATUS "using system libuv")
+elseif(DOWNLOAD_UV)
+  message(STATUS "using libuv submodule")
+  set(LIBUV_ROOT ${CMAKE_SOURCE_DIR}/external/libuv)
   add_subdirectory(${LIBUV_ROOT})
   set(LIBUV_INCLUDE_DIRS ${LIBUV_ROOT}/include)
   set(LIBUV_LIBRARY uv_a)
   add_definitions(-D_LARGEFILE_SOURCE)
   add_definitions(-D_FILE_OFFSET_BITS=64)
-elseif(NOT LIBUV_IN_SOURCE)
-  find_package(LibUV 1.28.0 REQUIRED)
 endif()
 
 include_directories(${LIBUV_INCLUDE_DIRS})

contrib/cross/aarch64.toolchain.cmake

@@ -0,0 +1,12 @@
+set(CMAKE_SYSTEM_NAME Linux)
+set(TOOLCHAIN_PREFIX aarch64-linux-gnu)
+#set(TOOLCHAIN_SUFFIX)
+
+set(CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX})
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+
+set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc${TOOLCHAIN_SUFFIX})
+set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++${TOOLCHAIN_SUFFIX})

external/libuv

@@ -0,0 +1 @@
+Subproject commit f52786b4badf82e1bfad06b52ade0397b61cd14e