Initial import

.gitignore

@@ -0,0 +1,6 @@
+obj/
+winobj/
+fastzip
+fastzip.exe
+*.zip
+

LICENSE.md

@@ -0,0 +1,37 @@
+# LICENSE
+
+## Fastzip license
+
+Fastzip is licensed under the MIT license.<br/>
+Copyright (c) 2015 Unity Technologies.<br/>
+see src/LICENCE<br/>
+
+## Licenses of used code
+
+### Crc32.cpp
+
+ZLib-like license<br/>
+Copyright (c) 2011-2015 Stephan Brumme. All rights reserved.<br/>
+Slicing-by-16 contributed by Bulat Ziganshin<br/>
+see http://create.stephan-brumme.com/disclaimer.html<br/>
+
+### IGzip
+
+MIT license<br/>
+Copyright (c) 2014 Intel Corporation<br/>
+see https://software.intel.com/en-us/articles/igzip-a-high-performance-deflate-compressor-with-optimizations-for-genomic-data<br/>
+
+### Infozip
+
+Info-ZIP license<br/>
+Copyright (c) 1990-2007 Info-ZIP.  All rights reserved.<br/>
+See src/infozip/LICENCE<br/>
+
+### OpenSSL
+
+Dual licensed<br/>
+Copyright (c) 1998-2011 The OpenSSL Project.  All rights reserved.<br/>
+Copyright (C) 1995-1998 Eric Young ([email protected])<br/>
+See src/openssl/LICENCE<br/>
+
+

Makefile

@@ -0,0 +1,160 @@
+# Set this to zero to compile without the Intel fast deflate
+# ...Also means you don't need YASM
+# WITH_INTEL=0
+# BENCHMARK=1
+
+ifeq ($(HOST),)
+
+ifeq ($(OS),Windows_NT)
+  HOST = windows
+  CMDEXE = 1
+else
+  UNAME_S := $(shell uname -s)
+  UNAME_N := $(shell uname -n)
+  ifeq ($(UNAME_S),Linux)
+    HOST = linux
+  endif
+  ifeq ($(UNAME_S),Darwin)
+    HOST = apple
+  endif
+endif
+
+endif
+
+# Basic settings
+LD = $(CXX)
+YASM = yasm
+OBJDIR = obj
+SRCDIR = src
+CFLAGS = -g -O2 -I$(SRCDIR)/igzip -I$(SRCDIR)/infozip -I$(SRCDIR)/openssl/include -Wno-deprecated-declarations -Wno-unused-result
+CFLAGS += -Wall -Wno-unused-variable -Wno-unused-function
+ASMFLAGS=-I $(SRCDIR)/igzip
+LIBS = -lcrypto
+TARGET = fastzip
+
+ifeq ($(BENCHMARK),1)
+  CFLAGS += -DDO_BENCHMARK -I/usr/local/include
+  LIBS += -lbenchmark
+  LDFLAGS += -L/usr/local/lib
+endif
+
+ifeq ($(HOST),windows)
+  CFLAGS += -mno-ms-bitfields -I$(SRCDIR)/mingw-std-threads
+  ASMFLAGS += -DWIN_CC=1
+  LIBS += -lgdi32 
+  LDFLAGS += -Lprebuilt/win -static
+  TARGET := $(TARGET).exe
+  OBJDIR = winobj
+  ifeq ($(WITH_INTEL),)
+    WITH_INTEL=0
+  endif
+else
+  ifeq ($(WITH_INTEL),)
+    WITH_INTEL=1
+  endif
+endif
+
+ifeq ($(HOST),apple)
+  ASMFLAGS += -f macho64
+else
+  ASMFLAGS += -f elf64
+endif
+
+ifeq ($(HOST),linux)
+  LDFLAGS+=-pthread
+endif
+
+CXXFLAGS=$(CFLAGS) -std=c++11
+
+#CXX=g++
+#CC=gcc
+LD=$(CXX)
+
+OBJFILES= \
+  $(OBJDIR)/fastzip_keystore.o \
+  $(OBJDIR)/inflate.o \
+  $(OBJDIR)/main.o \
+  $(OBJDIR)/utils.o \
+  $(OBJDIR)/fastzip.o \
+  $(OBJDIR)/asn.o \
+  $(OBJDIR)/crypto.o \
+  $(OBJDIR)/crc32/Crc32.o \
+  $(OBJDIR)/infozip.o \
+  $(OBJDIR)/infozip/deflate.o \
+  $(OBJDIR)/infozip/trees.o
+
+ifeq ($(WITH_INTEL),1)
+  OBJFILES += \
+    $(OBJDIR)/igzip/igzip1c_body.o \
+    $(OBJDIR)/igzip/igzip1c_finish.o \
+    $(OBJDIR)/igzip/c_code/common.o \
+    $(OBJDIR)/igzip/c_code/crc.o \
+    $(OBJDIR)/igzip/c_code/crc_utils.o \
+    $(OBJDIR)/igzip/c_code/hufftables_c.o \
+    $(OBJDIR)/igzip/bitbuf2.o \
+    $(OBJDIR)/igzip/crc.o \
+    $(OBJDIR)/igzip/huffman.o \
+    $(OBJDIR)/igzip/hufftables.o \
+    $(OBJDIR)/igzip/init_stream.o \
+    $(OBJDIR)/igzip/utils.o
+
+  CFLAGS += -DWITH_INTEL
+endif
+
+VERSION=_`date "+%y%m%d"`
+
+all : $(TARGET)
+
+dist :
+	$(MAKE) -C .
+	WITH_INTEL=0 HOST=windows PREFIX=x86_64-w64-mingw32- $(MAKE) -C .
+	strip fastzip
+	x86_64-w64-mingw32-strip fastzip.exe
+	./fastzip fastzip$(VERSION).zip fastzip.exe=win/fastzip.exe fastzip=mac/fastzip
+
+
+$(OBJDIR)/main.o : $(SRCDIR)/utils.h $(SRCDIR)/fastzip.h
+$(OBJDIR)/fastzip.o : $(SRCDIR)/utils.h $(SRCDIR)/ziparchive.h $(SRCDIR)/fastzip.h $(SRCDIR)/crypto.h $(SRCDIR)/asn.h
+$(OBJDIR)/crypto.o : $(SRCDIR)/crypto.h
+
+# Not used currently
+ifeq ($(CMDEXE),1)
+
+clean:
+	@del /Q /S $(OBJDIR)
+	@del /Q /S $(TARGET)
+
+$(OBJDIR)/%.o: $(SRCDIR)/%.c
+	@if not exist "$(@D)" @mkdir "$(@D)"
+	$(CC) -c $(CFLAGS) $< -o $@
+
+$(OBJDIR)/%.o: $(SRCDIR)/%.cpp
+	@if not exist "$(@D)" @mkdir "$(@D)"
+	$(CXX) -c $(CXXFLAGS) $< -o $@
+
+$(OBJDIR)/%.o: $(SRCDIR)/%.asm
+	@if not exist "$(@D)" @mkdir "$(@D)"
+	$(YASM) $(ASMFLAGS) $< -o $@
+
+else
+
+clean:
+	rm -f $(OBJFILES) ${TARGET}
+
+$(OBJDIR)/%.o: $(SRCDIR)/%.c
+	@mkdir -p $(@D)
+	$(PREFIX)$(CC) -c $(CFLAGS) $< -o $@
+
+$(OBJDIR)/%.o: $(SRCDIR)/%.cpp
+	@mkdir -p $(@D)
+	$(PREFIX)$(CXX) -c $(CXXFLAGS) $< -o $@
+
+$(OBJDIR)/%.o: $(SRCDIR)/%.asm
+	@mkdir -p $(@D)
+	$(YASM) $(ASMFLAGS) $< -o $@
+
+endif
+
+$(TARGET): $(OBJFILES) $(LIBMODS) $(DEPS)
+	$(PREFIX)$(LD) -o $(TARGET) $(OBJFILES) $(LIBMODS) $(LIBS) $(LDFLAGS)
+

README.md

@@ -0,0 +1,108 @@
+# Fastzip
+by _Jonas Minnberg_ ([email protected])
+
+* Parallell zip compression using *Info-ZIP* deflate or *Intel* fast deflate
+* On-the-fly Jar signing
+* Created with the goal of fast APK creation.
+
+* Check LICENCE.md for license
+
+## Build (Unix/OSX)
+
+* Make sure you have *yasm* and *openssl* installed.
+* `make`
+* `WITH_INTEL=0 make` - Build without Intel code
+* `./test.py` - Simple functional test
+
+## Build (Windows)
+
+* Make sure you have a working MingW compiler setup. [STLs Mingw distro](http://nuwen.net/mingw.html) should
+  work out of the box.
+* Build from _CMD.EXE_, If you have _sh.exe_ in your path it will not work.
+* `make`
+
+## Usage
+
+    fastzip <directory>
+    fastzip <file.zip> <paths>...
+
+## Speed Tests
+
+### Simple Commandline Test
+
+    # time fastzip android-ndk_auto-r10e
+    fastzip android-ndk_auto-r10e  140.39s user 4.87s system 739% cpu 19.647 total
+    # time zip -5 -r -q ndk.zip android-ndk_auto-r10e
+    zip -5 -r -q ndk.zip android-ndk_auto-r10e  88.72s user 1.89s system 98% cpu 1:31.82 total
+    # ls -l *.zip
+    -rw-r--r--    1 sasq  staff  1105140127 Oct 26 10:46 android-ndk_auto-r10e.zip
+    -rw-r--r--    1 sasq  staff  1108973881 Oct 26 10:48 ndk.zip
+
+### Unity Test (Unity Teleporter Demo)
+
+Export APK from Unity. Timed from Progress Dialog open until it closes. Resulting APK ~300MB
+
+### WINDOWS (i7 3.50GHz 8core)
+* Without Fastzip: *7:30min*
+* With Fastzip: *19s*
+* Constant overhead (staging area setup and moving APK) ~10s
+* Speedup: *~23x* overall (actual APK creation *~48x*)
+
+### MAC OS X (i7 2.8GHz 8core)
+* Without Fastzip: *7:52min*
+* With Fastzip: *30s*
+* Constant overhead (staging area setup and moving APK) ~20s
+* Speedup: *~16x* overall (actual APK creation *~46x*)
+
+## Old Tests
+
+Note - timings are made after at least one recursion through the directory to fill the cache.
+
+### Test 1 - Geometry, textures & other game data
+
+    # du -h test/ | tail -n 1
+    2.8G    test/
+
+#### Intel fast deflate (default)
+    # time ./fastzip test.zip test
+    ./fastzip test.zip test  61.16s user 7.43s system 648% cpu 10.572 total
+    # ls -lh test.zip
+    -rw-r--r--  1 jonasm  staff   2.6G Mar 14 23:30 test.zip
+
+#### Info-Zip deflate
+    # time ./fastzip -z test.zip test
+    ./fastzip -m infozip test.zip test  130.60s user 7.93s system 706% cpu 19.606 total
+    # ls -lh test.zip
+    -rw-r--r--  1 jonasm  staff   2.5G Mar 14 23:29 test.zip
+
+#### Intel fast deflate with early out storing of uncompressable files
+    # time ./fastzip -e test.zip test
+    ./fastzip -e test.zip test  16.55s user 6.64s system 392% cpu 5.909 total
+    # ls -lh test.zip
+    -rw-r--r--  1 jonasm  staff   2.6G Mar 14 23:30 test.zip
+
+#### Compare: Info zip with same data and compression level
+    # time zip -q -1 -r test.zip test
+    zip -q -1 -r test.zip test  77.98s user 1.85s system 97% cpu 1:21.47 total
+    # ls -lh test.zip
+    -rw-r--r--  1 jonasm  staff   2.5G Mar 15 23:01 test.zip
+
+### Test 2 - The Android NDK directory
+
+    # du -h android-ndk-r10b | tail -n 1
+    1.5G    android-ndk-r10b
+
+    # time fastzip android.zip android-ndk-r10b
+    fastzip android.zip android-ndk-r10b  11.55s user 3.38s system 527% cpu 2.831 total
+    # ls -lh android.zip
+    -rw-r--r--  1 jonasm  staff   719M Mar 17 09:09 android.zip
+
+    # time fastzip -z android.zip android-ndk-r10b
+    fastzip -z android.zip android-ndk-r10b  32.13s user 3.50s system 651% cpu 5.470 total
+    # ls -lh android.zip
+    -rw-r--r--  1 jonasm  staff   625M Mar 17 09:10 android.zip
+
+    # time zip -q -1 -r androidz.zip android-ndk-r10b
+    zip -q -1 -r androidz.zip android-ndk-r10b  26.16s user 1.34s system 97% cpu 28.067 total
+    # ls -lh androidz.zip
+    -rw-r--r--  1 jonasm  staff   628M Mar 17 09:11 androidz.zip

src/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Unity Technologies, Jonas Minnberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.