Skip to content

Commit

Permalink
thirdparty: Add xxhash
Browse files Browse the repository at this point in the history
  • Loading branch information
dmex committed Jan 7, 2025
1 parent f48dde8 commit ff5354e
Show file tree
Hide file tree
Showing 15 changed files with 7,888 additions and 23 deletions.
4 changes: 2 additions & 2 deletions SystemInformer/srvprv.c
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ ULONG PhpServiceHashtableHashFunction(
{
PPH_SERVICE_ITEM serviceItem = *(PPH_SERVICE_ITEM *)Entry;

return PhHashStringRefEx(&serviceItem->Key, TRUE, PH_STRING_HASH_X65599);
return PhHashStringRefEx(&serviceItem->Key, TRUE, PH_STRING_HASH_XXH32);
}

PPH_SERVICE_ITEM PhpLookupServiceItem(
Expand Down Expand Up @@ -440,7 +440,7 @@ static ULONG PhHashServiceNameEntry(
_In_ PPHP_SERVICE_NAME_ENTRY Value
)
{
return PhHashStringRefEx(&Value->Name, TRUE, PH_STRING_HASH_X65599);
return PhHashStringRefEx(&Value->Name, TRUE, PH_STRING_HASH_XXH32);
}

VOID PhServiceQueryStage1(
Expand Down
15 changes: 13 additions & 2 deletions phlib/basesup.c
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,9 @@

#include <phbase.h>
#include <phintrnl.h>
#include <phnative.h>
#include <phnativeinl.h>
#include <phintrin.h>
#include <circbuf.h>
#include <thirdparty.h>
#include <ntintsafe.h>

#ifndef PH_NATIVE_STRING_CONVERSION
Expand Down Expand Up @@ -6155,6 +6154,18 @@ ULONG PhHashStringRefEx(

return hash;
}
case PH_STRING_HASH_XXH32:
{
if (IgnoreCase)
{
NOTHING;
}

if (String->Length == 0)
return 0;

return PhHashStringRefXXH32(String, 0);
}
}

return 0;
Expand Down
2 changes: 1 addition & 1 deletion phlib/cpysave.c
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ PPH_LIST PhaFormatTextTable(
{
PH_STRING_BUILDER stringBuilder;

PhInitializeStringBuilder(&stringBuilder, 100);
PhInitializeStringBuilder(&stringBuilder, 0x100);

switch (Mode)
{
Expand Down
1 change: 1 addition & 0 deletions phlib/include/phbasesup.h
Original file line number Diff line number Diff line change
Expand Up @@ -3623,6 +3623,7 @@ typedef enum _PH_STRING_HASH
PH_STRING_HASH_DEFAULT,
PH_STRING_HASH_FNV1A,
PH_STRING_HASH_X65599,
PH_STRING_HASH_XXH32,
} PH_STRING_HASH;

PHLIBAPI
Expand Down
6 changes: 6 additions & 0 deletions phlib/include/thirdparty.h
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,9 @@
#else
#error "ThirdParty.lib is mssing"
#endif

#if __has_include("../../tools/thirdparty/xxhash/xxhashwrapper.h")
#include "../../tools/thirdparty/xxhash/xxhashwrapper.h"
#else
#error "ThirdParty.lib is mssing"
#endif
29 changes: 13 additions & 16 deletions phlib/native.c
Original file line number Diff line number Diff line change
Expand Up @@ -6544,7 +6544,7 @@ BOOLEAN NTAPI PhpIsDotNetEnumProcessModulesCallback(

typedef struct _PHP_PIPE_NAME_HASH
{
PPH_STRINGREF Name;
ULONG NameHash;
ULONG Found;
} PHP_PIPE_NAME_HASH, *PPHP_PIPE_NAME_HASH;

Expand All @@ -6560,10 +6560,7 @@ static BOOLEAN NTAPI PhpDotNetCorePipeHashCallback(
objectName.Length = Information->FileNameLength;
objectName.Buffer = Information->FileName;

if (
PhHashStringRefEx(context->Name, FALSE, PH_STRING_HASH_X65599) ==
PhHashStringRefEx(&objectName, FALSE, PH_STRING_HASH_X65599)
)
if (PhHashStringRefEx(&objectName, FALSE, PH_STRING_HASH_XXH32) == context->NameHash)
{
context->Found = TRUE;
return FALSE;
Expand Down Expand Up @@ -6603,7 +6600,7 @@ NTSTATUS PhGetProcessIsDotNetEx(
SIZE_T returnLength;
OBJECT_ATTRIBUTES objectAttributes;
UNICODE_STRING objectName;
PH_STRINGREF objectNameSr;
PH_STRINGREF objectNameStringRef;
PH_FORMAT format[2];
WCHAR formatBuffer[0x80];

Expand All @@ -6621,10 +6618,10 @@ NTSTATUS PhGetProcessIsDotNetEx(

if (PhFormatToBuffer(format, RTL_NUMBER_OF(format), formatBuffer, sizeof(formatBuffer), &returnLength))
{
objectNameSr.Length = returnLength - sizeof(UNICODE_NULL);
objectNameSr.Buffer = formatBuffer;
objectNameStringRef.Length = returnLength - sizeof(UNICODE_NULL);
objectNameStringRef.Buffer = formatBuffer;

PhStringRefToUnicodeString(&objectNameSr, &objectName);
PhStringRefToUnicodeString(&objectNameStringRef, &objectName);
}
else
{
Expand Down Expand Up @@ -6665,10 +6662,10 @@ NTSTATUS PhGetProcessIsDotNetEx(

if (PhFormatToBuffer(format, RTL_NUMBER_OF(format), formatBuffer, sizeof(formatBuffer), &returnLength))
{
objectNameSr.Length = returnLength - sizeof(UNICODE_NULL);
objectNameSr.Buffer = formatBuffer;
objectNameStringRef.Length = returnLength - sizeof(UNICODE_NULL);
objectNameStringRef.Buffer = formatBuffer;

PhStringRefToUnicodeString(&objectNameSr, &objectName);
PhStringRefToUnicodeString(&objectNameStringRef, &objectName);
}
else
{
Expand Down Expand Up @@ -6711,10 +6708,10 @@ NTSTATUS PhGetProcessIsDotNetEx(
{
PHP_PIPE_NAME_HASH context;

objectNameSr.Length = returnLength - sizeof(UNICODE_NULL);
objectNameSr.Buffer = formatBuffer;
PhStringRefToUnicodeString(&objectNameSr, &objectName);
context.Name = &objectNameSr;
objectNameStringRef.Length = returnLength - sizeof(UNICODE_NULL);
objectNameStringRef.Buffer = formatBuffer;
PhStringRefToUnicodeString(&objectNameStringRef, &objectName);
context.NameHash = PhHashStringRefEx(&objectNameStringRef, FALSE, PH_STRING_HASH_XXH32);
context.Found = FALSE;

status = PhEnumDirectoryNamedPipe(
Expand Down
6 changes: 5 additions & 1 deletion tools/thirdparty/thirdparty.vcxproj
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Project DefaultTargets="Build" ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|ARM64">
<Configuration>Debug</Configuration>
Expand Down Expand Up @@ -27,6 +27,7 @@
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<VCProjectVersion>17.0</VCProjectVersion>
<ProjectGuid>{60F7AED7-5094-41B2-A2C9-545398060C83}</ProjectGuid>
<RootNamespace>thirdparty</RootNamespace>
<Keyword>Win32Proj</Keyword>
Expand Down Expand Up @@ -223,6 +224,7 @@
<ClCompile Include="tlsh\tlsh_impl.cpp" />
<ClCompile Include="tlsh\tlsh_util.cpp" />
<ClCompile Include="tlsh\tlsh_wrapper.cpp" />
<ClCompile Include="xxhash\xxhashwrapper.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="detours\detours.h" />
Expand Down Expand Up @@ -275,6 +277,8 @@
<ClInclude Include="tlsh\tlsh_win_version.h" />
<ClInclude Include="tlsh\tlsh_wrapper.h" />
<ClInclude Include="tlsh\win_version.h" />
<ClInclude Include="xxhash\xxhashwrapper.h" />
<ClInclude Include="xxhash\xxhash.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
</Project>
14 changes: 13 additions & 1 deletion tools/thirdparty/thirdparty.vcxproj.filters
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Mini-XML">
<UniqueIdentifier>{7d0c2da2-811d-4844-9704-0c87067c7be1}</UniqueIdentifier>
Expand Down Expand Up @@ -43,6 +43,9 @@
<Filter Include="detours">
<UniqueIdentifier>{8e0a677b-267b-49fa-ad3a-4c80f411cc2b}</UniqueIdentifier>
</Filter>
<Filter Include="xxhash">
<UniqueIdentifier>{065de9df-0b97-4a3d-a7dc-b90b332859a8}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="jsonc\arraylist.c">
Expand Down Expand Up @@ -255,6 +258,9 @@
<ClCompile Include="mxml\mxml-options.c">
<Filter>Mini-XML</Filter>
</ClCompile>
<ClCompile Include="xxhash\xxhashwrapper.c">
<Filter>xxhash</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="mxml\config.h">
Expand Down Expand Up @@ -407,5 +413,11 @@
<ClInclude Include="mxml\mxml-private.h">
<Filter>Mini-XML\Headers</Filter>
</ClInclude>
<ClInclude Include="xxhash\xxhash.h">
<Filter>xxhash</Filter>
</ClInclude>
<ClInclude Include="xxhash\xxhashwrapper.h">
<Filter>xxhash</Filter>
</ClInclude>
</ItemGroup>
</Project>
4 changes: 4 additions & 0 deletions tools/thirdparty/tlsh/tlsh_wrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@

#pragma once

EXTERN_C_START

EXTERN_C
BOOLEAN
NTAPI
Expand All @@ -27,3 +29,5 @@ PvGetTlshFileHash(
_In_ HANDLE FileHandle,
_Out_ char** HashResult
);

EXTERN_C_END
104 changes: 104 additions & 0 deletions tools/thirdparty/xxhash/CHANGELOG
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
v0.8.3
- fix : variant `XXH3_128bits_withSecretandSeed()` could produce an invalid result in some specific set of conditions, #894 by @hltj
- cli : vector extension detected at runtime on x86/x64, enabled by default
- cli : new commands `--filelist` and `--files-from`, by @Ian-Clowes
- cli : XXH3 64-bits GNU format can now be generated and checked (command `-H3`)
- portability: LoongArch SX SIMD extension, by @lrzlin
- portability: can build on AIX, suggested by @likema
- portability: validated for SPARC cpus

v0.8.2
- fix : XXH3 S390x vector implementation (@hzhuang1)
- fix : PowerPC vector compilation with IBM XL compiler (@MaxiBoether)
- perf : improved WASM speed by x2/x3 using SIMD128 (@easyaspi314)
- perf : improved speed (+20%) for XXH3 on ARM NEON (@easyaspi314)
- cli : Fix filename contain /LF character (@t-mat)
- cli : Support # comment lines in --check files (@t-mat)
- cli : Support commands --binary and --ignore-missing (@t-mat)
- build: fix -Og compilation (@easyaspi314, @t-mat)
- build: fix pkgconfig generation with cmake (@ilya-fedin)
- build: fix icc compilation
- build: fix cmake install directories
- build: new build options XXH_NO_XXH3, XXH_SIZE_OPT and XXH_NO_STREAM to reduce binary size (@easyaspi314)
- build: dedicated install targets (@ffontaine)
- build: support DISPATCH mode in cmake (@hzhuang1)
- portability: fix x86dispatch when building with Visual + clang-cl (@t-mat)
- portability: SVE vector implementation of XXH3 (@hzhuang1)
- portability: compatibility with freestanding environments, using XXH_NO_STDLIB
- portability: can build on Haiku (@Begasus)
- portability: validated on m68k and risc-v
- doc : XXH3 specification (@Adrien1018)
- doc : improved doxygen documentation (@easyaspi314, @t-mat)
- misc : dedicated sanity test binary (@t-mat)

v0.8.1
- perf : much improved performance for XXH3 streaming variants, notably on gcc and msvc
- perf : improved XXH64 speed and latency on small inputs
- perf : small XXH32 speed and latency improvement on small inputs of random size
- perf : minor stack usage improvement for XXH32 and XXH64
- api : new experimental variants XXH3_*_withSecretandSeed()
- api : update XXH3_generateSecret(), can no generate secret of any size (>= XXH3_SECRET_SIZE_MIN)
- cli : xxhsum can now generate and check XXH3 checksums, using command `-H3`
- build: can build xxhash without XXH3, with new build macro XXH_NO_XXH3
- build: fix xxh_x86dispatch build with MSVC, by @apankrat
- build: XXH_INLINE_ALL can always be used safely, even after XXH_NAMESPACE or a previous XXH_INLINE_ALL
- build: improved PPC64LE vector support, by @mpe
- install: fix pkgconfig, by @ellert
- install: compatibility with Haiku, by @Begasus
- doc : code comments made compatible with doxygen, by @easyaspi314
- misc : XXH_ACCEPT_NULL_INPUT_POINTER is no longer necessary, all functions can accept NULL input pointers, as long as size == 0
- misc : complete refactor of CI tests on Github Actions, offering much larger coverage, by @t-mat
- misc : xxhsum code base split into multiple specialized units, within directory cli/, by @easyaspi314

v0.8.0
- api : stabilize XXH3
- cli : xxhsum can parse BSD-style --check lines, by @WayneD
- cli : `xxhsum -` accepts console input, requested by @jaki
- cli : xxhsum accepts -- separator, by @jaki
- cli : fix : print correct default algo for symlinked helpers, by @martinetd
- install: improved pkgconfig script, allowing custom install locations, requested by @ellert

v0.7.4
- perf: automatic vector detection and selection at runtime (`xxh_x86dispatch.h`), initiated by @easyaspi314
- perf: added AVX512 support, by @gzm55
- api : new: secret generator `XXH_generateSecret()`, suggested by @koraa
- api : fix: XXH3_state_t is movable, identified by @koraa
- api : fix: state is correctly aligned in AVX mode (unlike `malloc()`), by @easyaspi314
- api : fix: streaming generated wrong values in some combination of random ingestion lengths, reported by @WayneD
- cli : fix unicode print on Windows, by @easyaspi314
- cli : can `-c` check file generated by sfv
- build: `make DISPATCH=1` generates `xxhsum` and `libxxhash` with runtime vector detection (x86/x64 only)
- install: cygwin installation support
- doc : Cryptol specification of XXH32 and XXH64, by @weaversa

v0.7.3
- perf: improved speed for large inputs (~+20%)
- perf: improved latency for small inputs (~10%)
- perf: s390x Vectorial code, by @easyaspi314
- cli: improved support for Unicode filenames on Windows, thanks to @easyaspi314 and @t-mat
- api: `xxhash.h` can now be included in any order, with and without `XXH_STATIC_LINKING_ONLY` and `XXH_INLINE_ALL`
- build: xxHash's implementation transferred into `xxhash.h`. No more need to have `xxhash.c` in the `/include` directory for `XXH_INLINE_ALL` to work
- install: created pkg-config file, by @bket
- install: VCpkg installation instructions, by @LilyWangL
- doc: Highly improved code documentation, by @easyaspi314
- misc: New test tool in `/tests/collisions`: brute force collision tester for 64-bit hashes

v0.7.2
- Fixed collision ratio of `XXH128` for some specific input lengths, reported by @svpv
- Improved `VSX` and `NEON` variants, by @easyaspi314
- Improved performance of scalar code path (`XXH_VECTOR=0`), by @easyaspi314
- `xxhsum`: can generate 128-bit hashes with the `-H2` option (note: for experimental purposes only! `XXH128` is not yet frozen)
- `xxhsum`: option `-q` removes status notifications

v0.7.1
- Secret first: the algorithm computation can be altered by providing a "secret", which is any blob of bytes, of size >= `XXH3_SECRET_SIZE_MIN`.
- `seed` is still available, and acts as a secret generator
- updated `ARM NEON` variant by @easyaspi314
- Streaming implementation is available
- Improve compatibility and performance with Visual Studio, with help from @aras-p
- Better integration when using `XXH_INLINE_ALL`: do not pollute host namespace, use its own macros, such as `XXH_ASSERT()`, `XXH_ALIGN`, etc.
- 128-bit variant provides helper functions for comparison of hashes.
- Better `clang` generation of `rotl` instruction, thanks to @easyaspi314
- `XXH_REROLL` build macro to reduce binary size, by @easyaspi314
- Improved `cmake` script, by @Mezozoysky
- Full benchmark program provided in `/tests/bench`
26 changes: 26 additions & 0 deletions tools/thirdparty/xxhash/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
xxHash Library
Copyright (c) 2012-2021 Yann Collet
All rights reserved.

BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Loading

0 comments on commit ff5354e

Please sign in to comment.