Skip to content

Commit

Permalink
Merge pull request #782 from apernet/wip-geosite
Browse files Browse the repository at this point in the history
feat: GeoSite support & reworked GeoIP
  • Loading branch information
tobyxdd authored Oct 29, 2023
2 parents fcc3dd4 + e604c12 commit affe092
Showing 24 changed files with 34,568 additions and 221 deletions.
28 changes: 15 additions & 13 deletions app/cmd/server.go
Original file line number Diff line number Diff line change
@@ -143,9 +143,10 @@ type serverConfigResolver struct {
}

type serverConfigACL struct {
File string `mapstructure:"file"`
Inline []string `mapstructure:"inline"`
GeoIP string `mapstructure:"geoip"`
File string `mapstructure:"file"`
Inline []string `mapstructure:"inline"`
GeoIP string `mapstructure:"geoip"`
GeoSite string `mapstructure:"geosite"`
}

type serverConfigOutboundDirect struct {
@@ -460,21 +461,22 @@ func (c *serverConfig) fillOutboundConfig(hyConfig *server.Config) error {
if c.ACL.File != "" && len(c.ACL.Inline) > 0 {
return configError{Field: "acl", Err: errors.New("cannot set both acl.file and acl.inline")}
}
gLoader := &utils.GeoIPLoader{
Filename: c.ACL.GeoIP,
DownloadFunc: geoipDownloadFunc,
DownloadErrFunc: geoipDownloadErrFunc,
gLoader := &utils.GeoLoader{
GeoIPFilename: c.ACL.GeoIP,
GeoSiteFilename: c.ACL.GeoSite,
DownloadFunc: geoDownloadFunc,
DownloadErrFunc: geoDownloadErrFunc,
}
if c.ACL.File != "" {
hasACL = true
acl, err := outbounds.NewACLEngineFromFile(c.ACL.File, obs, gLoader.Load)
acl, err := outbounds.NewACLEngineFromFile(c.ACL.File, obs, gLoader)
if err != nil {
return configError{Field: "acl.file", Err: err}
}
uOb = acl
} else if len(c.ACL.Inline) > 0 {
hasACL = true
acl, err := outbounds.NewACLEngineFromString(strings.Join(c.ACL.Inline, "\n"), obs, gLoader.Load)
acl, err := outbounds.NewACLEngineFromString(strings.Join(c.ACL.Inline, "\n"), obs, gLoader)
if err != nil {
return configError{Field: "acl.inline", Err: err}
}
@@ -764,13 +766,13 @@ func runMasqTCPServer(s *masq.MasqTCPServer, httpAddr, httpsAddr string) {
}
}

func geoipDownloadFunc(filename, url string) {
logger.Info("downloading GeoIP database", zap.String("filename", filename), zap.String("url", url))
func geoDownloadFunc(filename, url string) {
logger.Info("downloading database", zap.String("filename", filename), zap.String("url", url))
}

func geoipDownloadErrFunc(err error) {
func geoDownloadErrFunc(err error) {
if err != nil {
logger.Error("failed to download GeoIP database", zap.Error(err))
logger.Error("failed to download database", zap.Error(err))
}
}

3 changes: 2 additions & 1 deletion app/cmd/server_test.go
Original file line number Diff line number Diff line change
@@ -101,7 +101,8 @@ func TestServerConfig(t *testing.T) {
"lmao(ok)",
"kek(cringe,boba,tea)",
},
GeoIP: "fake.mmdb",
GeoIP: "some.dat",
GeoSite: "some_site.dat",
},
Outbounds: []serverConfigOutboundEntry{
{
3 changes: 2 additions & 1 deletion app/cmd/server_test.yaml
Original file line number Diff line number Diff line change
@@ -75,7 +75,8 @@ acl:
inline:
- lmao(ok)
- kek(cringe,boba,tea)
geoip: fake.mmdb
geoip: some.dat
geosite: some_site.dat

outbounds:
- name: goodstuff
3 changes: 1 addition & 2 deletions app/go.mod
Original file line number Diff line number Diff line change
@@ -9,7 +9,6 @@ require (
github.com/caddyserver/certmagic v0.17.2
github.com/mdp/qrterminal/v3 v3.1.1
github.com/mholt/acmez v1.0.4
github.com/oschwald/geoip2-golang v1.9.0
github.com/spf13/cobra v1.7.0
github.com/spf13/viper v1.15.0
github.com/stretchr/testify v1.8.4
@@ -33,7 +32,6 @@ require (
github.com/miekg/dns v1.1.55 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/onsi/ginkgo/v2 v2.9.5 // indirect
github.com/oschwald/maxminddb-golang v1.11.0 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -56,6 +54,7 @@ require (
golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/tools v0.11.1 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
rsc.io/qr v0.2.0 // indirect
7 changes: 3 additions & 4 deletions app/go.sum
Original file line number Diff line number Diff line change
@@ -102,6 +102,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -115,6 +116,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -179,10 +181,6 @@ github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc=
github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y=
github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0=
github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
@@ -575,6 +573,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
70 changes: 0 additions & 70 deletions app/internal/utils/geoip.go

This file was deleted.

107 changes: 107 additions & 0 deletions app/internal/utils/geoloader.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package utils

import (
"io"
"net/http"
"os"

"github.com/apernet/hysteria/extras/outbounds/acl"
"github.com/apernet/hysteria/extras/outbounds/acl/v2geo"
)

const (
geoipFilename = "geoip.dat"
geoipURL = "https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geoip.dat"
geositeFilename = "geosite.dat"
geositeURL = "https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@release/geosite.dat"
)

var _ acl.GeoLoader = (*GeoLoader)(nil)

// GeoLoader provides the on-demand GeoIP/GeoSite database
// loading functionality required by the ACL engine.
// Empty filenames = automatic download from built-in URLs.
type GeoLoader struct {
GeoIPFilename string
GeoSiteFilename string

DownloadFunc func(filename, url string)
DownloadErrFunc func(err error)

geoipMap map[string]*v2geo.GeoIP
geositeMap map[string]*v2geo.GeoSite
}

func (l *GeoLoader) download(filename, url string) error {
l.DownloadFunc(filename, url)

resp, err := http.Get(url)
if err != nil {
l.DownloadErrFunc(err)
return err
}
defer resp.Body.Close()

f, err := os.Create(filename)
if err != nil {
l.DownloadErrFunc(err)
return err
}
defer f.Close()

_, err = io.Copy(f, resp.Body)
l.DownloadErrFunc(err)
return err
}

func (l *GeoLoader) LoadGeoIP() (map[string]*v2geo.GeoIP, error) {
if l.geoipMap != nil {
return l.geoipMap, nil
}
autoDL := false
filename := l.GeoIPFilename
if filename == "" {
autoDL = true
filename = geoipFilename
}
m, err := v2geo.LoadGeoIP(filename)
if os.IsNotExist(err) && autoDL {
// It's ok, we will download it.
err = l.download(filename, geoipURL)
if err != nil {
return nil, err
}
m, err = v2geo.LoadGeoIP(filename)
}
if err != nil {
return nil, err
}
l.geoipMap = m
return m, nil
}

func (l *GeoLoader) LoadGeoSite() (map[string]*v2geo.GeoSite, error) {
if l.geositeMap != nil {
return l.geositeMap, nil
}
autoDL := false
filename := l.GeoSiteFilename
if filename == "" {
autoDL = true
filename = geositeFilename
}
m, err := v2geo.LoadGeoSite(filename)
if os.IsNotExist(err) && autoDL {
// It's ok, we will download it.
err = l.download(filename, geositeURL)
if err != nil {
return nil, err
}
m, err = v2geo.LoadGeoSite(filename)
}
if err != nil {
return nil, err
}
l.geositeMap = m
return m, nil
}
3 changes: 1 addition & 2 deletions extras/go.mod
Original file line number Diff line number Diff line change
@@ -7,10 +7,10 @@ require (
github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6
github.com/hashicorp/golang-lru/v2 v2.0.5
github.com/miekg/dns v1.1.55
github.com/oschwald/geoip2-golang v1.9.0
github.com/stretchr/testify v1.8.4
github.com/txthinking/socks5 v0.0.0-20230325130024-4230056ae301
golang.org/x/crypto v0.14.0
google.golang.org/protobuf v1.28.1
)

require (
@@ -19,7 +19,6 @@ require (
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
github.com/onsi/ginkgo/v2 v2.9.5 // indirect
github.com/oschwald/maxminddb-golang v1.11.0 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
8 changes: 4 additions & 4 deletions extras/go.sum
Original file line number Diff line number Diff line change
@@ -12,8 +12,10 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
@@ -32,10 +34,6 @@ github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc=
github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y=
github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0=
github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -108,6 +106,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
golang.org/x/tools v0.11.1 h1:ojD5zOW8+7dOGzdnNgersm8aPfcDjhMp12UfG93NIMc=
golang.org/x/tools v0.11.1/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Loading

0 comments on commit affe092

Please sign in to comment.