From 6cf3affd02d6b3406c4011db4e50d38d33b09ba5 Mon Sep 17 00:00:00 2001 From: laiwei Date: Tue, 20 Nov 2018 19:41:26 +0800 Subject: [PATCH 01/10] run test case in docker --- .travis.yml | 19 +++++++++++++--- Makefile | 4 ++++ docker_test.sh | 43 ++++++++++++++++++++++++++++++++++++ modules/api/test/api_test.go | 2 +- 4 files changed, 64 insertions(+), 4 deletions(-) create mode 100755 docker_test.sh diff --git a/.travis.yml b/.travis.yml index b3cc0825d..4dc45f90b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,28 @@ +sudo: required + language: go go: - - 1.6.x - - 1.7.x - - 1.8.x + - "1.8" + +env: + - DB_USER=root + - DB_PASSWORD=test123456 + - DB_HOST=127.0.0.1 + - DB_PORT=13306 + - REDIS_HOST=127.0.0.1 + - REDIS_PORT=16379 + +services: + - docker script: - make install - make fmt-check - make misspell-check - make all + - bash ./docker_test.sh + after_success: - go test -race -coverprofile=coverage.txt -covermode=atomic github.com/open-falcon/falcon-plus/modules/api/test diff --git a/Makefile b/Makefile index 2ace784c0..773a45964 100644 --- a/Makefile +++ b/Makefile @@ -89,6 +89,10 @@ pack4docker: checkbin tar -C out -zcf open-falcon-v$(VERSION).tar.gz . @rm -rf out +.PHONY: test +test: + @go test ./modules/api/test + clean: @rm -rf ./bin @rm -rf ./out diff --git a/docker_test.sh b/docker_test.sh new file mode 100755 index 000000000..f0190ce13 --- /dev/null +++ b/docker_test.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +export DB_USER=root +export DB_PASSWORD=test123456 +export DB_HOST=127.0.0.1 +export DB_PORT=13306 +export REDIS_HOST=127.0.0.1 +export REDIS_PORT=16379 +export API_PORT=18080 +export API_HOST=127.0.0.1 + +docker rm -f falcon-mysql falcon-redis falcon-plus &> /dev/null +docker run --name falcon-mysql -e MYSQL_ROOT_PASSWORD=$DB_PASSWORD -p $DB_PORT:3306 -d mysql:5.7 +docker run --name falcon-redis -p $REDIS_PORT:6379 -d redis:4-alpine3.8 + +echo "waiting mysql start..." +sleep 10 +for x in `ls ./scripts/mysql/db_schema/*.sql`; do + echo "- - -" $x ... + mysql -h $DB_HOST -P$DB_PORT -u$DB_USER -p$DB_PASSWORD < $x +done + +commit_id=`git rev-parse --short HEAD` +image_tag="falcon-plus:$commit_id" + +#build docker image from source code +docker build -t $image_tag . + +## run falcon-plus container +docker run -itd --name falcon-plus \ + --link=falcon-mysql:db.falcon \ + --link=falcon-redis:redis.falcon \ + -p 18433:8433 \ + -p 18080:8080 \ + -e MYSQL_PORT=$DB_USER:$DB_PASSWORD@tcp\(db.falcon:3306\) \ + -e REDIS_PORT=redis.falcon:6379 \ + $image_tag + +## start falcon backend modules, such as graph,api,etc. +docker exec falcon-plus sh ctrl.sh start \ + graph hbs judge transfer nodata aggregator agent gateway api alarm + +make test diff --git a/modules/api/test/api_test.go b/modules/api/test/api_test.go index 794edb536..d77b324e0 100755 --- a/modules/api/test/api_test.go +++ b/modules/api/test/api_test.go @@ -44,7 +44,7 @@ var ( func init() { cfg_file := os.Getenv("API_TEST_CFG") if cfg_file == "" { - cfg_file = "./cfg_example" + cfg_file = "./cfg.example" } viper.SetConfigName(cfg_file) viper.AddConfigPath(".") From 9d3d33b3f8e2b6cd1f4427c484e40f6f3bf63e8f Mon Sep 17 00:00:00 2001 From: laiwei Date: Tue, 20 Nov 2018 23:04:11 +0800 Subject: [PATCH 02/10] fix env in .travis-ci config --- .travis.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4dc45f90b..3eb61c8eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,12 +6,7 @@ go: - "1.8" env: - - DB_USER=root - - DB_PASSWORD=test123456 - - DB_HOST=127.0.0.1 - - DB_PORT=13306 - - REDIS_HOST=127.0.0.1 - - REDIS_PORT=16379 + - DB_USER=root DB_PASSWORD=test123456 DB_HOST=127.0.0.1 DB_PORT=13306 REDIS_HOST=127.0.0.1 REDIS_PORT=16379 API_PORT=18080 API_HOST=127.0.0.1 services: - docker From 3089ce3079c1825ad7bdb05144385daab34b444b Mon Sep 17 00:00:00 2001 From: laiwei Date: Sun, 14 Apr 2019 22:34:42 +0800 Subject: [PATCH 03/10] add api doc for update_specific_user --- docs/_posts/User/2017-01-01-user_update.md | 1 - .../User/2019-04-14-update-specific-user.md | 29 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 docs/_posts/User/2019-04-14-update-specific-user.md diff --git a/docs/_posts/User/2017-01-01-user_update.md b/docs/_posts/User/2017-01-01-user_update.md index 371e9b208..b0ca3f33a 100644 --- a/docs/_posts/User/2017-01-01-user_update.md +++ b/docs/_posts/User/2017-01-01-user_update.md @@ -12,7 +12,6 @@ layout: default ### Request ```{ - "name": "test1", "cnname": "翱鶚Test", "email": "root123@cepave.com", "im": "44955834958", diff --git a/docs/_posts/User/2019-04-14-update-specific-user.md b/docs/_posts/User/2019-04-14-update-specific-user.md new file mode 100644 index 000000000..5d03d0970 --- /dev/null +++ b/docs/_posts/User/2019-04-14-update-specific-user.md @@ -0,0 +1,29 @@ +--- +category: User +apiurl: '/api/v1/user/u/:uid' +title: 'Update Specific User' +type: 'PUT' +sample_doc: 'user.html' +layout: default +--- + +更新使用者 +* [Session](#/authentication) Required + +### Request +```{ + "cnname": "翱鶚Test", + "email": "root123@cepave.com", + "im": "44955834958", + "phone": "99999999999", + "qq": "904394234239" +}``` + +### Response + +```Status: 200``` +```{"message":"user info updated"}``` + +For more example, see the [user](/doc/user.html). + +For errors responses, see the [response status codes documentation](#/response-status-codes). From 0d3360d6ef354e61a801d39345ae73fbc735bb9a Mon Sep 17 00:00:00 2001 From: laiwei Date: Tue, 30 Apr 2019 11:34:59 +0800 Subject: [PATCH 04/10] code fmt check --- modules/api/app/controller/alarm/alarm_events_controller.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/api/app/controller/alarm/alarm_events_controller.go b/modules/api/app/controller/alarm/alarm_events_controller.go index 1c5d22f79..ccb743f7e 100644 --- a/modules/api/app/controller/alarm/alarm_events_controller.go +++ b/modules/api/app/controller/alarm/alarm_events_controller.go @@ -111,7 +111,7 @@ func AlarmLists(c *gin.Context) { alarmDB := inputs.collectDBFilters(db.Alarm, f.TableName(), nil) cevens := []alm.EventCases{} //if no specific, will give return first 2000 records - if inputs.Page == -1 && inputs.Limit == -1{ + if inputs.Page == -1 && inputs.Limit == -1 { inputs.Limit = 2000 alarmDB = alarmDB.Order("timestamp DESC").Limit(inputs.Limit) } else if inputs.Limit == -1 { @@ -140,7 +140,7 @@ func AlarmLists(c *gin.Context) { if inputs.Limit >= 50 { inputs.Limit = 50 } - step := (inputs.Page -1) * inputs.Limit + step := (inputs.Page - 1) * inputs.Limit alarmDB = alarmDB.Order("timestamp DESC").Offset(step).Limit(inputs.Limit) } alarmDB.Find(&cevens) @@ -194,7 +194,7 @@ func EventsGet(c *gin.Context) { if inputs.Limit <= 0 || inputs.Limit >= 50 { inputs.Limit = 50 } - step := (inputs.Page -1) * inputs.Limit + step := (inputs.Page - 1) * inputs.Limit eventDB.Order("timestamp DESC").Offset(step).Limit(inputs.Limit).Scan(&evens) h.JSONR(c, evens) } From 7ba08bdcf497cea14d0753958c33fce140c07a3f Mon Sep 17 00:00:00 2001 From: laiwei Date: Mon, 13 May 2019 18:25:16 +0800 Subject: [PATCH 05/10] use local time zone in mysql config --- config/alarm.json | 2 +- docker_test.sh | 2 ++ modules/alarm/cfg.example.json | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/alarm.json b/config/alarm.json index 534c34228..94658ea1a 100644 --- a/config/alarm.json +++ b/config/alarm.json @@ -31,7 +31,7 @@ "plus_api_token": "%%PLUS_API_DEFAULT_TOKEN%%" }, "falcon_portal": { - "addr": "%%MYSQL%%/alarms?charset=utf8&loc=Asia%2FChongqing", + "addr": "%%MYSQL%%/alarms?charset=utf8&loc=Local", "idle": 10, "max": 100 }, diff --git a/docker_test.sh b/docker_test.sh index f0190ce13..f591034f3 100755 --- a/docker_test.sh +++ b/docker_test.sh @@ -37,7 +37,9 @@ docker run -itd --name falcon-plus \ $image_tag ## start falcon backend modules, such as graph,api,etc. +sleep 5 docker exec falcon-plus sh ctrl.sh start \ graph hbs judge transfer nodata aggregator agent gateway api alarm +sleep 5 make test diff --git a/modules/alarm/cfg.example.json b/modules/alarm/cfg.example.json index eb62d25c2..81d039541 100644 --- a/modules/alarm/cfg.example.json +++ b/modules/alarm/cfg.example.json @@ -31,7 +31,7 @@ "plus_api_token": "default-token-used-in-server-side" }, "falcon_portal": { - "addr": "root:@tcp(127.0.0.1:3306)/alarms?charset=utf8&loc=Asia%2FChongqing", + "addr": "root:@tcp(127.0.0.1:3306)/alarms?charset=utf8&loc=Local", "idle": 10, "max": 100 }, From b413e30b8ffbabfff67b78617111f5e5cfd5a9d4 Mon Sep 17 00:00:00 2001 From: laiwei Date: Thu, 30 May 2019 16:52:49 +0800 Subject: [PATCH 06/10] v0.3, bugfix and enhancement --- README.md | 2 +- VERSION | 2 +- docker/README.md | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 49001329e..6e53c49ca 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ mysql -h 127.0.0.1 -u root -p < 4_graph-db-schema.sql mysql -h 127.0.0.1 -u root -p < 5_alarms-db-schema.sql ``` -**NOTE: if you are upgrading from v0.1 to current version v0.2.0,then**. [More upgrading instruction](http://www.jianshu.com/p/6fb2c2b4d030) +**NOTE: if you are upgrading from v0.1 to v0.2.0(or above),then**. [More upgrading instruction](http://www.jianshu.com/p/6fb2c2b4d030) mysql -h 127.0.0.1 -u root -p < 5_alarms-db-schema.sql diff --git a/VERSION b/VERSION index 0c62199f1..be5863417 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.1 +0.3 diff --git a/docker/README.md b/docker/README.md index bea6cd42a..a613d3170 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,6 +1,6 @@ ## Running open-falcon container -`the latest version in docker hub is v0.2.1` +`the latest version in docker hub is v0.3` ##### 1. Start mysql and init the mysql table before the first running ``` @@ -33,7 +33,7 @@ docker run --name falcon-redis -p6379:6379 -d redis:4-alpine3.8 ``` ## pull images from hub.docker.com/openfalcon - docker pull openfalcon/falcon-plus:v0.2.1 + docker pull openfalcon/falcon-plus:v0.3 ## run falcon-plus container docker run -itd --name falcon-plus \ @@ -45,7 +45,7 @@ docker run --name falcon-redis -p6379:6379 -d redis:4-alpine3.8 -e REDIS_PORT=redis.falcon:6379 \ -v /home/work/open-falcon/data:/open-falcon/data \ -v /home/work/open-falcon/logs:/open-falcon/logs \ - openfalcon/falcon-plus:v0.2.1 + openfalcon/falcon-plus:v0.3 ## start falcon backend modules, such as graph,api,etc. docker exec falcon-plus sh ctrl.sh start \ @@ -93,7 +93,7 @@ docker run --name falcon-redis -p6379:6379 -d redis:4-alpine3.8 cd /tmp && \ git clone https://github.com/open-falcon/falcon-plus && \ cd /tmp/falcon-plus/ && \ - docker build -t falcon-plus:v0.2.1 . + docker build -t falcon-plus:v0.3 . ``` ##### Building falcon-dashboard From 4be9d31e67e919e987b8cb37b4a8f097938f76ac Mon Sep 17 00:00:00 2001 From: laiwei Date: Tue, 15 Oct 2019 11:53:03 +0800 Subject: [PATCH 07/10] make modules version more consistent --- Makefile | 4 ++-- VERSION | 2 +- main.go | 2 +- modules/agent/cron/reporter.go | 2 +- modules/agent/g/const.go | 12 +++++++++++- modules/agent/http/health.go | 2 +- modules/agent/main.go | 6 +++++- modules/agent/version.go | 22 ++++++++++++++++++++++ modules/aggregator/g/g.go | 10 ++++++++-- modules/aggregator/http/common.go | 2 +- modules/aggregator/main.go | 6 +++++- modules/aggregator/version.go | 22 ++++++++++++++++++++++ modules/alarm/g/g.go | 10 ++++++++-- modules/alarm/http/controller.go | 2 +- modules/alarm/main.go | 6 +++++- modules/alarm/version.go | 22 ++++++++++++++++++++++ modules/api/config/g.go | 11 ++++++++--- modules/api/main.go | 6 +++++- modules/api/version.go | 22 ++++++++++++++++++++++ modules/gateway/g/g.go | 11 ++++++++++- modules/gateway/http/common.go | 2 +- modules/gateway/main.go | 6 +++++- modules/gateway/version.go | 22 ++++++++++++++++++++++ modules/graph/g/g.go | 11 ++++++++++- modules/graph/http/common.go | 2 +- modules/graph/main.go | 8 ++++++-- modules/graph/version.go | 22 ++++++++++++++++++++++ modules/hbs/g/g.go | 11 +++++++++-- modules/hbs/http/common.go | 2 +- modules/hbs/main.go | 6 +++++- modules/hbs/version.go | 22 ++++++++++++++++++++++ modules/judge/g/g.go | 11 +++++++++-- modules/judge/http/common.go | 2 +- modules/judge/main.go | 6 +++++- modules/judge/version.go | 22 ++++++++++++++++++++++ modules/nodata/g/g.go | 10 ++++++++-- modules/nodata/http/common.go | 2 +- modules/nodata/main.go | 8 ++++++-- modules/nodata/version.go | 22 ++++++++++++++++++++++ modules/transfer/g/g.go | 11 ++++++++++- modules/transfer/http/common.go | 2 +- modules/transfer/main.go | 8 ++++++-- modules/transfer/version.go | 22 ++++++++++++++++++++++ version.go | 5 +++-- 44 files changed, 381 insertions(+), 46 deletions(-) create mode 100644 modules/agent/version.go create mode 100644 modules/aggregator/version.go create mode 100644 modules/alarm/version.go create mode 100644 modules/api/version.go create mode 100644 modules/gateway/version.go create mode 100644 modules/graph/version.go create mode 100644 modules/hbs/version.go create mode 100644 modules/judge/version.go create mode 100644 modules/nodata/version.go create mode 100644 modules/transfer/version.go diff --git a/Makefile b/Makefile index 773a45964..f205ad8d3 100644 --- a/Makefile +++ b/Makefile @@ -44,11 +44,11 @@ fmt-check: fi; $(CMD): - go build -o bin/$@/falcon-$@ ./modules/$@ + go build -ldflags "-X main.BinaryName=$@ -X main.GitCommit=`git rev-parse --short HEAD` -X main.Version=$(VERSION)" -o bin/$@/falcon-$@ ./modules/$@ .PHONY: $(TARGET) $(TARGET): $(GOFILES) - go build -ldflags "-X main.GitCommit=`git rev-parse --short HEAD` -X main.Version=$(VERSION)" -o open-falcon + go build -ldflags "-X main.BinaryName=Open-Falcon -X main.GitCommit=`git rev-parse --short HEAD` -X main.Version=$(VERSION)" -o open-falcon checkbin: bin/ config/ open-falcon diff --git a/VERSION b/VERSION index be5863417..adf50c2c6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3 +0.3.x diff --git a/main.go b/main.go index da33da848..c830cfec7 100644 --- a/main.go +++ b/main.go @@ -28,7 +28,7 @@ var RootCmd = &cobra.Command{ Use: "open-falcon", RunE: func(c *cobra.Command, args []string) error { if versionFlag { - fmt.Printf("Open-Falcon version %s, build %s\n", Version, GitCommit) + fmt.Printf("%s version %s, build %s\n", BinaryName, Version, GitCommit) return nil } return c.Usage() diff --git a/modules/agent/cron/reporter.go b/modules/agent/cron/reporter.go index 3593dad6f..3002f9727 100644 --- a/modules/agent/cron/reporter.go +++ b/modules/agent/cron/reporter.go @@ -38,7 +38,7 @@ func reportAgentStatus(interval time.Duration) { req := model.AgentReportRequest{ Hostname: hostname, IP: g.IP(), - AgentVersion: g.VERSION, + AgentVersion: g.VersionMsg(), PluginVersion: g.GetCurrPluginVersion(), } diff --git a/modules/agent/g/const.go b/modules/agent/g/const.go index 41d133ed6..5cefe6427 100644 --- a/modules/agent/g/const.go +++ b/modules/agent/g/const.go @@ -25,8 +25,18 @@ import ( // 5.1.0: 同步插件的时候不再使用checksum机制 // 5.1.1: 修复往多个transfer发送数据的时候crash的问题 // 5.1.2: ignore mount point when blocks=0 + +var ( + BinaryName string + Version string + GitCommit string +) + +func VersionMsg() string { + return Version + "@" + GitCommit +} + const ( - VERSION = "5.1.2" COLLECT_INTERVAL = time.Second URL_CHECK_HEALTH = "url.check.health" NET_PORT_LISTEN = "net.port.listen" diff --git a/modules/agent/http/health.go b/modules/agent/http/health.go index 4982bf326..2d135bb30 100644 --- a/modules/agent/http/health.go +++ b/modules/agent/http/health.go @@ -25,6 +25,6 @@ func configHealthRoutes() { }) http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(g.VERSION)) + w.Write([]byte(g.VersionMsg())) }) } diff --git a/modules/agent/main.go b/modules/agent/main.go index 0b0c16b4b..96cddce37 100644 --- a/modules/agent/main.go +++ b/modules/agent/main.go @@ -26,6 +26,10 @@ import ( func main() { + g.BinaryName = BinaryName + g.Version = Version + g.GitCommit = GitCommit + cfg := flag.String("c", "cfg.json", "configuration file") version := flag.Bool("v", false, "show version") check := flag.Bool("check", false, "check collector") @@ -33,7 +37,7 @@ func main() { flag.Parse() if *version { - fmt.Println(g.VERSION) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } diff --git a/modules/agent/version.go b/modules/agent/version.go new file mode 100644 index 000000000..48137e37a --- /dev/null +++ b/modules/agent/version.go @@ -0,0 +1,22 @@ +// Copyright 2017 Xiaomi, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// build !release +package main + +var ( + Version = "" + GitCommit = "" + BinaryName = "" +) diff --git a/modules/aggregator/g/g.go b/modules/aggregator/g/g.go index 23fedccd5..103b9bb85 100644 --- a/modules/aggregator/g/g.go +++ b/modules/aggregator/g/g.go @@ -19,10 +19,16 @@ import ( "runtime" ) -const ( - VERSION = "0.0.4" +var ( + BinaryName string + Version string + GitCommit string ) +func VersionMsg() string { + return Version + "@" + GitCommit +} + func init() { runtime.GOMAXPROCS(runtime.NumCPU()) log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) diff --git a/modules/aggregator/http/common.go b/modules/aggregator/http/common.go index 38b9c3eed..71d7ef250 100644 --- a/modules/aggregator/http/common.go +++ b/modules/aggregator/http/common.go @@ -27,7 +27,7 @@ func configCommonRoutes() { }) http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(g.VERSION)) + w.Write([]byte(g.VersionMsg())) }) http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) { diff --git a/modules/aggregator/main.go b/modules/aggregator/main.go index 20a524d03..a82adc30e 100644 --- a/modules/aggregator/main.go +++ b/modules/aggregator/main.go @@ -29,13 +29,17 @@ import ( ) func main() { + g.BinaryName = BinaryName + g.Version = Version + g.GitCommit = GitCommit + cfg := flag.String("c", "cfg.json", "configuration file") version := flag.Bool("v", false, "show version") help := flag.Bool("h", false, "help") flag.Parse() if *version { - fmt.Println(g.VERSION) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } diff --git a/modules/aggregator/version.go b/modules/aggregator/version.go new file mode 100644 index 000000000..48137e37a --- /dev/null +++ b/modules/aggregator/version.go @@ -0,0 +1,22 @@ +// Copyright 2017 Xiaomi, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// build !release +package main + +var ( + Version = "" + GitCommit = "" + BinaryName = "" +) diff --git a/modules/alarm/g/g.go b/modules/alarm/g/g.go index 54fc3eba2..103b9bb85 100644 --- a/modules/alarm/g/g.go +++ b/modules/alarm/g/g.go @@ -19,10 +19,16 @@ import ( "runtime" ) -const ( - VERSION = "0.2.0" +var ( + BinaryName string + Version string + GitCommit string ) +func VersionMsg() string { + return Version + "@" + GitCommit +} + func init() { runtime.GOMAXPROCS(runtime.NumCPU()) log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) diff --git a/modules/alarm/http/controller.go b/modules/alarm/http/controller.go index 1ba8d9fcb..4867e1dd4 100644 --- a/modules/alarm/http/controller.go +++ b/modules/alarm/http/controller.go @@ -21,7 +21,7 @@ import ( ) func Version(c *gin.Context) { - c.String(200, g.VERSION) + c.String(200, g.VersionMsg()) } func Health(c *gin.Context) { diff --git a/modules/alarm/main.go b/modules/alarm/main.go index 4ed7669bb..325e764ae 100644 --- a/modules/alarm/main.go +++ b/modules/alarm/main.go @@ -29,13 +29,17 @@ import ( ) func main() { + g.BinaryName = BinaryName + g.Version = Version + g.GitCommit = GitCommit + cfg := flag.String("c", "cfg.json", "configuration file") version := flag.Bool("v", false, "show version") help := flag.Bool("h", false, "help") flag.Parse() if *version { - fmt.Println(g.VERSION) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } diff --git a/modules/alarm/version.go b/modules/alarm/version.go new file mode 100644 index 000000000..48137e37a --- /dev/null +++ b/modules/alarm/version.go @@ -0,0 +1,22 @@ +// Copyright 2017 Xiaomi, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// build !release +package main + +var ( + Version = "" + GitCommit = "" + BinaryName = "" +) diff --git a/modules/api/config/g.go b/modules/api/config/g.go index 1e2b18262..139d71b5c 100644 --- a/modules/api/config/g.go +++ b/modules/api/config/g.go @@ -19,11 +19,16 @@ import ( "runtime" ) -// change log: -const ( - VERSION = "0.0.1" +var ( + BinaryName string + Version string + GitCommit string ) +func VersionMsg() string { + return Version + "@" + GitCommit +} + func init() { runtime.GOMAXPROCS(runtime.NumCPU()) log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) diff --git a/modules/api/main.go b/modules/api/main.go index 2e4f86511..95606bc4a 100644 --- a/modules/api/main.go +++ b/modules/api/main.go @@ -37,13 +37,17 @@ func initGraph() { } func main() { + config.BinaryName = BinaryName + config.Version = Version + config.GitCommit = GitCommit + cfgTmp := flag.String("c", "cfg.json", "configuration file") version := flag.Bool("v", false, "show version") help := flag.Bool("h", false, "help") flag.Parse() cfg := *cfgTmp if *version { - fmt.Println(config.VERSION) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } diff --git a/modules/api/version.go b/modules/api/version.go new file mode 100644 index 000000000..48137e37a --- /dev/null +++ b/modules/api/version.go @@ -0,0 +1,22 @@ +// Copyright 2017 Xiaomi, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// build !release +package main + +var ( + Version = "" + GitCommit = "" + BinaryName = "" +) diff --git a/modules/gateway/g/g.go b/modules/gateway/g/g.go index 96f7a94d1..56d71deb1 100644 --- a/modules/gateway/g/g.go +++ b/modules/gateway/g/g.go @@ -31,8 +31,17 @@ import ( // 0.0.10: control sending concurrent of slow transfers // 0.0.11: use pfc +var ( + BinaryName string + Version string + GitCommit string +) + +func VersionMsg() string { + return Version + "@" + GitCommit +} + const ( - VERSION = "0.0.11" GAUGE = "GAUGE" COUNTER = "COUNTER" DERIVE = "DERIVE" diff --git a/modules/gateway/http/common.go b/modules/gateway/http/common.go index 39586b1cb..452d79792 100644 --- a/modules/gateway/http/common.go +++ b/modules/gateway/http/common.go @@ -30,7 +30,7 @@ func configCommonRoutes() { }) http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION))) + w.Write([]byte(fmt.Sprintf("%s\n", g.VersionMsg()))) }) http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) { diff --git a/modules/gateway/main.go b/modules/gateway/main.go index 50049978b..c37a31af4 100644 --- a/modules/gateway/main.go +++ b/modules/gateway/main.go @@ -26,12 +26,16 @@ import ( ) func main() { + g.BinaryName = BinaryName + g.Version = Version + g.GitCommit = GitCommit + cfg := flag.String("c", "cfg.json", "configuration file") version := flag.Bool("v", false, "show version") flag.Parse() if *version { - fmt.Println(g.VERSION) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } diff --git a/modules/gateway/version.go b/modules/gateway/version.go new file mode 100644 index 000000000..48137e37a --- /dev/null +++ b/modules/gateway/version.go @@ -0,0 +1,22 @@ +// Copyright 2017 Xiaomi, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// build !release +package main + +var ( + Version = "" + GitCommit = "" + BinaryName = "" +) diff --git a/modules/graph/g/g.go b/modules/graph/g/g.go index 7443659b0..4a5aa1ae6 100644 --- a/modules/graph/g/g.go +++ b/modules/graph/g/g.go @@ -38,8 +38,17 @@ import ( // 0.5.8 clean GraphItems/historyCache Cache at regular intervals // 0.5.9 add flush style(flush by number of every counter's monitoring data) +var ( + BinaryName string + Version string + GitCommit string +) + +func VersionMsg() string { + return Version + "@" + GitCommit +} + const ( - VERSION = "0.5.9" GAUGE = "GAUGE" DERIVE = "DERIVE" COUNTER = "COUNTER" diff --git a/modules/graph/http/common.go b/modules/graph/http/common.go index abca0cd38..132fd3e09 100644 --- a/modules/graph/http/common.go +++ b/modules/graph/http/common.go @@ -35,7 +35,7 @@ func configCommonRoutes() { }) router.GET("/api/v2/version", func(c *gin.Context) { - JSONR(c, 200, gin.H{"value": g.VERSION}) + JSONR(c, 200, gin.H{"value": g.VersionMsg()}) }) router.GET("/api/v2/workdir", func(c *gin.Context) { diff --git a/modules/graph/main.go b/modules/graph/main.go index b9902e2b4..4467aa76a 100644 --- a/modules/graph/main.go +++ b/modules/graph/main.go @@ -65,17 +65,21 @@ func start_signal(pid int, cfg *g.GlobalConfig) { } func main() { + g.BinaryName = BinaryName + g.Version = Version + g.GitCommit = GitCommit + cfg := flag.String("c", "cfg.json", "specify config file") version := flag.Bool("v", false, "show version") versionGit := flag.Bool("vg", false, "show version and git commit log") flag.Parse() if *version { - fmt.Println(g.VERSION) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } if *versionGit { - fmt.Println(g.VERSION, g.COMMIT) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } diff --git a/modules/graph/version.go b/modules/graph/version.go new file mode 100644 index 000000000..48137e37a --- /dev/null +++ b/modules/graph/version.go @@ -0,0 +1,22 @@ +// Copyright 2017 Xiaomi, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// build !release +package main + +var ( + Version = "" + GitCommit = "" + BinaryName = "" +) diff --git a/modules/hbs/g/g.go b/modules/hbs/g/g.go index 09db12832..52d8e4522 100644 --- a/modules/hbs/g/g.go +++ b/modules/hbs/g/g.go @@ -24,10 +24,17 @@ import ( // 1.0.8: bugfix loop init cache // 1.0.9: update host table anyway // 1.1.0: remove Checksum when query plugins -const ( - VERSION = "1.1.0" + +var ( + BinaryName string + Version string + GitCommit string ) +func VersionMsg() string { + return Version + "@" + GitCommit +} + func init() { runtime.GOMAXPROCS(runtime.NumCPU()) log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) diff --git a/modules/hbs/http/common.go b/modules/hbs/http/common.go index c301364b5..d3f71de01 100644 --- a/modules/hbs/http/common.go +++ b/modules/hbs/http/common.go @@ -27,7 +27,7 @@ func configCommonRoutes() { }) http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(g.VERSION)) + w.Write([]byte(g.VersionMsg())) }) http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) { diff --git a/modules/hbs/main.go b/modules/hbs/main.go index 03f3295d9..53ed964db 100644 --- a/modules/hbs/main.go +++ b/modules/hbs/main.go @@ -28,12 +28,16 @@ import ( ) func main() { + g.BinaryName = BinaryName + g.Version = Version + g.GitCommit = GitCommit + cfg := flag.String("c", "cfg.json", "configuration file") version := flag.Bool("v", false, "show version") flag.Parse() if *version { - fmt.Println(g.VERSION) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } diff --git a/modules/hbs/version.go b/modules/hbs/version.go new file mode 100644 index 000000000..48137e37a --- /dev/null +++ b/modules/hbs/version.go @@ -0,0 +1,22 @@ +// Copyright 2017 Xiaomi, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// build !release +package main + +var ( + Version = "" + GitCommit = "" + BinaryName = "" +) diff --git a/modules/judge/g/g.go b/modules/judge/g/g.go index a5ea4e92c..c072d2180 100644 --- a/modules/judge/g/g.go +++ b/modules/judge/g/g.go @@ -23,10 +23,17 @@ import ( // 2.0.1: bugfix HistoryData limit // 2.0.2: clean stale data // 2.0.3: add timeout to sync strategies and expressions -const ( - VERSION = "2.0.3" + +var ( + BinaryName string + Version string + GitCommit string ) +func VersionMsg() string { + return Version + "@" + GitCommit +} + func init() { runtime.GOMAXPROCS(runtime.NumCPU()) log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) diff --git a/modules/judge/http/common.go b/modules/judge/http/common.go index 94d7ebbc7..ffbf28995 100644 --- a/modules/judge/http/common.go +++ b/modules/judge/http/common.go @@ -27,7 +27,7 @@ func configCommonRoutes() { }) http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(g.VERSION)) + w.Write([]byte(g.VersionMsg())) }) http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) { diff --git a/modules/judge/main.go b/modules/judge/main.go index 5ec3fa558..85dbcb56a 100644 --- a/modules/judge/main.go +++ b/modules/judge/main.go @@ -26,12 +26,16 @@ import ( ) func main() { + g.BinaryName = BinaryName + g.Version = Version + g.GitCommit = GitCommit + cfg := flag.String("c", "cfg.json", "configuration file") version := flag.Bool("v", false, "show version") flag.Parse() if *version { - fmt.Println(g.VERSION) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } diff --git a/modules/judge/version.go b/modules/judge/version.go new file mode 100644 index 000000000..48137e37a --- /dev/null +++ b/modules/judge/version.go @@ -0,0 +1,22 @@ +// Copyright 2017 Xiaomi, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// build !release +package main + +var ( + Version = "" + GitCommit = "" + BinaryName = "" +) diff --git a/modules/nodata/g/g.go b/modules/nodata/g/g.go index cd63f1a1d..b96248ac5 100644 --- a/modules/nodata/g/g.go +++ b/modules/nodata/g/g.go @@ -29,10 +29,16 @@ import ( // 0.0.7 use gauss distribution to get threshold, sync judge and sender, fix bug of collector's cache // 0.0.8 simplify project -const ( - VERSION = "0.0.8" +var ( + BinaryName string + Version string + GitCommit string ) +func VersionMsg() string { + return Version + "@" + GitCommit +} + func init() { runtime.GOMAXPROCS(runtime.NumCPU()) log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) diff --git a/modules/nodata/http/common.go b/modules/nodata/http/common.go index 1e63ccb07..d5953621a 100644 --- a/modules/nodata/http/common.go +++ b/modules/nodata/http/common.go @@ -30,7 +30,7 @@ func configCommonRoutes() { }) http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION))) + w.Write([]byte(fmt.Sprintf("%s\n", g.VersionMsg()))) }) http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) { diff --git a/modules/nodata/main.go b/modules/nodata/main.go index 6752e1ba1..325007564 100644 --- a/modules/nodata/main.go +++ b/modules/nodata/main.go @@ -27,17 +27,21 @@ import ( ) func main() { + g.BinaryName = BinaryName + g.Version = Version + g.GitCommit = GitCommit + cfg := flag.String("c", "cfg.json", "configuration file") version := flag.Bool("v", false, "show version") versionGit := flag.Bool("vg", false, "show version") flag.Parse() if *version { - fmt.Println(g.VERSION) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } if *versionGit { - fmt.Println(g.VERSION, g.COMMIT) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } diff --git a/modules/nodata/version.go b/modules/nodata/version.go new file mode 100644 index 000000000..48137e37a --- /dev/null +++ b/modules/nodata/version.go @@ -0,0 +1,22 @@ +// Copyright 2017 Xiaomi, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// build !release +package main + +var ( + Version = "" + GitCommit = "" + BinaryName = "" +) diff --git a/modules/transfer/g/g.go b/modules/transfer/g/g.go index c21dc15ae..ea95c91c4 100644 --- a/modules/transfer/g/g.go +++ b/modules/transfer/g/g.go @@ -31,8 +31,17 @@ import ( // 0.0.16: support config of min step // 0.0.17: remove migrating, which is implemented in graph +var ( + BinaryName string + Version string + GitCommit string +) + +func VersionMsg() string { + return Version + "@" + GitCommit +} + const ( - VERSION = "0.0.17" GAUGE = "GAUGE" COUNTER = "COUNTER" DERIVE = "DERIVE" diff --git a/modules/transfer/http/common.go b/modules/transfer/http/common.go index 1b5602dae..dfea6cd6a 100644 --- a/modules/transfer/http/common.go +++ b/modules/transfer/http/common.go @@ -28,7 +28,7 @@ func configCommonRoutes() { }) http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION))) + w.Write([]byte(fmt.Sprintf("%s\n", g.VersionMsg()))) }) http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) { diff --git a/modules/transfer/main.go b/modules/transfer/main.go index 8839c07db..9d2d54bf8 100644 --- a/modules/transfer/main.go +++ b/modules/transfer/main.go @@ -26,17 +26,21 @@ import ( ) func main() { + g.BinaryName = BinaryName + g.Version = Version + g.GitCommit = GitCommit + cfg := flag.String("c", "cfg.json", "configuration file") version := flag.Bool("v", false, "show version") versionGit := flag.Bool("vg", false, "show version") flag.Parse() if *version { - fmt.Println(g.VERSION) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } if *versionGit { - fmt.Println(g.VERSION, g.COMMIT) + fmt.Printf("Open-Falcon %s version %s, build %s\n", BinaryName, Version, GitCommit) os.Exit(0) } diff --git a/modules/transfer/version.go b/modules/transfer/version.go new file mode 100644 index 000000000..48137e37a --- /dev/null +++ b/modules/transfer/version.go @@ -0,0 +1,22 @@ +// Copyright 2017 Xiaomi, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// build !release +package main + +var ( + Version = "" + GitCommit = "" + BinaryName = "" +) diff --git a/version.go b/version.go index d44996987..48137e37a 100644 --- a/version.go +++ b/version.go @@ -16,6 +16,7 @@ package main var ( - Version = "" - GitCommit = "" + Version = "" + GitCommit = "" + BinaryName = "" ) From feb307260a6d1e41bdb6a20a59e87b63de9a08fd Mon Sep 17 00:00:00 2001 From: laiwei Date: Fri, 14 Feb 2020 22:24:57 +0800 Subject: [PATCH 08/10] refine migrate --- modules/api/test/api_test.go | 66 ++++++++++---- modules/graph/api/graph.go | 8 +- modules/graph/api/rpc.go | 8 +- modules/graph/cron/clean.go | 6 +- modules/graph/g/cfg.go | 12 +-- modules/graph/g/db.go | 6 +- modules/graph/g/g.go | 3 - modules/graph/http/common.go | 17 ++++ modules/graph/http/http.go | 15 --- modules/graph/index/cache.go | 6 +- modules/graph/main.go | 21 +++-- modules/graph/rrdtool/migrate.go | 53 +++++------ modules/graph/rrdtool/rrdtool.go | 142 ++++++++++++++++++++++------- modules/graph/rrdtool/sync_disk.go | 21 +++-- modules/graph/store/store.go | 29 +++--- 15 files changed, 263 insertions(+), 150 deletions(-) diff --git a/modules/api/test/api_test.go b/modules/api/test/api_test.go index 13bc75a29..4cb7f9bc1 100755 --- a/modules/api/test/api_test.go +++ b/modules/api/test/api_test.go @@ -20,6 +20,7 @@ import ( "os" "strings" "testing" + "time" "github.com/masato25/resty" log "github.com/sirupsen/logrus" @@ -555,31 +556,60 @@ func TestGraph(t *testing.T) { rc := resty.New() rc.SetHeader("Apitoken", api_token) + var rr *[]map[string]interface{} = &[]map[string]interface{}{} Convey("Get endpoint list: GET /graph/endpoint", t, func() { - r := []map[string]interface{}{} resp, _ := rc.R().SetQueryParam("q", ".+"). - SetResult(&r). + SetResult(rr). Get(fmt.Sprintf("%s/graph/endpoint", api_v1)) So(resp.StatusCode(), ShouldEqual, 200) - So(len(r), ShouldBeGreaterThanOrEqualTo, 0) + So(len(*rr), ShouldBeGreaterThanOrEqualTo, 0) - if len(r) == 0 { - return - } + }) - eid := r[0]["id"] - r = []map[string]interface{}{} - Convey("Get counter list: GET /graph/endpoint_counter", func() { - resp, _ := rc.R(). - SetQueryParam("eid", fmt.Sprintf("%v", eid)). - SetQueryParam("metricQuery", ".+"). - SetQueryParam("limit", "1"). - SetResult(&r). - Get(fmt.Sprintf("%s/graph/endpoint_counter", api_v1)) - So(resp.StatusCode(), ShouldEqual, 200) - So(r, ShouldNotBeEmpty) - }) + if len(*rr) == 0 { + return + } + + eid := (*rr)[0]["id"] + endpoint := (*rr)[0]["endpoint"] + Convey("Get counter list: GET /graph/endpoint_counter", t, func() { + resp, _ := rc.R(). + SetQueryParam("eid", fmt.Sprintf("%v", eid)). + SetQueryParam("metricQuery", ".+"). + SetQueryParam("limit", "1"). + SetResult(rr). + Get(fmt.Sprintf("%s/graph/endpoint_counter", api_v1)) + So(resp.StatusCode(), ShouldEqual, 200) + So(*rr, ShouldNotBeEmpty) + }) + + if len(*rr) == 0 { + return + } + + counter := (*rr)[0]["counter"] + step := (*rr)[0]["step"] + + now := time.Now() + start_ts := now.Add(time.Duration(-1) * time.Hour).Unix() + end_ts := now.Unix() + + Convey("Query counter history: POST /graph/history", t, func() { + resp, _ := rc.R(). + SetBody(map[string]interface{}{ + "step": step, + "consol_fun": "AVERAGE", + "start_time": start_ts, + "end_time": end_ts, + "hostnames": []string{endpoint.(string)}, + "counters": []string{counter.(string)}, + }). + SetResult(rr). + Post(fmt.Sprintf("%s/graph/history", api_v1)) + log.Info(resp) + So(resp.StatusCode(), ShouldEqual, 200) + So(*rr, ShouldNotBeEmpty) }) } diff --git a/modules/graph/api/graph.go b/modules/graph/api/graph.go index 55b628f31..a91e931a8 100644 --- a/modules/graph/api/graph.go +++ b/modules/graph/api/graph.go @@ -48,7 +48,7 @@ func (this *Graph) GetRrd(key string, rrdfile *g.File) (err error) { items := store.GraphItems.PopAll(key) if len(items) > 0 { - rrdtool.FlushFile(rrdfile.Filename, md5, items) + rrdtool.CommitFile(rrdfile.Filename, md5, items) } rrdfile.Body, err = rrdtool.ReadFile(rrdfile.Filename, md5) @@ -178,7 +178,11 @@ func (this *Graph) Query(param cmodel.GraphQueryParam, resp *cmodel.GraphQueryRe // read data from rrd file // 从RRD中获取数据不包含起始时间点 // 例: start_ts=1484651400,step=60,则第一个数据时间为1484651460) - datas, _ = rrdtool.Fetch(filename, md5, param.ConsolFun, start_ts-int64(step), end_ts, step) + var err error + datas, err = rrdtool.Fetch(filename, md5, param.ConsolFun, start_ts-int64(step), end_ts, step) + if err != nil { + log.Debugf("rrdtool fetch %s error: %v", filename, err) + } datas_size = len(datas) } diff --git a/modules/graph/api/rpc.go b/modules/graph/api/rpc.go index fbb3aef4f..3cf1b289f 100644 --- a/modules/graph/api/rpc.go +++ b/modules/graph/api/rpc.go @@ -16,7 +16,7 @@ package api import ( "container/list" - "log" + log "github.com/sirupsen/logrus" "net" "net/rpc" "sync" @@ -52,7 +52,7 @@ func init() { func Start() { if !g.Config().Rpc.Enabled { - log.Println("rpc.Start warning, not enabled") + log.Info("rpc.Start warning, not enabled") return } addr := g.Config().Rpc.Listen @@ -65,7 +65,7 @@ func Start() { if err != nil { log.Fatalf("rpc.Start error, listen %s failed, %s", addr, err) } else { - log.Println("rpc.Start ok, listening on", addr) + log.Info("rpc.Start ok, listening on", addr) } rpc.Register(new(Graph)) @@ -99,7 +99,7 @@ func Start() { select { case <-Close_chan: - log.Println("rpc, recv sigout and exiting...") + log.Info("rpc, recv sigout and exiting...") listener.Close() Close_done_chan <- 1 diff --git a/modules/graph/cron/clean.go b/modules/graph/cron/clean.go index 515dc7538..dbcd0ec9d 100644 --- a/modules/graph/cron/clean.go +++ b/modules/graph/cron/clean.go @@ -29,7 +29,7 @@ package cron import ( - "log" + log "github.com/sirupsen/logrus" "strings" "time" @@ -85,7 +85,7 @@ func DeleteInvalidItems() int { pfc.Gauge("GraphItemsCacheCnt", int64(currentCnt)) pfc.Gauge("GraphItemsCacheInvalidCnt", int64(deleteCnt)) - log.Printf("GraphItemsCache: Count=>%d, DeleteInvalid=>%d", currentCnt, deleteCnt) + log.Infof("GraphItemsCache: Count=>%d, DeleteInvalid=>%d", currentCnt, deleteCnt) return deleteCnt } @@ -107,7 +107,7 @@ func DeleteInvalidHistory() int { pfc.Gauge("HistoryCacheCnt", int64(currentCnt)) pfc.Gauge("HistoryCacheInvalidCnt", int64(deleteCnt)) - log.Printf("HistoryCache: Count=>%d, DeleteInvalid=>%d", currentCnt, deleteCnt) + log.Infof("HistoryCache: Count=>%d, DeleteInvalid=>%d", currentCnt, deleteCnt) return deleteCnt } diff --git a/modules/graph/g/cfg.go b/modules/graph/g/cfg.go index e8b1021b3..85efa0153 100644 --- a/modules/graph/g/cfg.go +++ b/modules/graph/g/cfg.go @@ -16,7 +16,7 @@ package g import ( "encoding/json" - "log" + log "github.com/sirupsen/logrus" "strconv" "sync/atomic" "unsafe" @@ -77,24 +77,24 @@ func Config() *GlobalConfig { func ParseConfig(cfg string) { if cfg == "" { - log.Fatalln("config file not specified: use -c $filename") + log.Fatal("config file not specified: use -c $filename") } if !file.IsExist(cfg) { - log.Fatalln("config file specified not found:", cfg) + log.Fatal("config file specified not found:", cfg) } ConfigFile = cfg configContent, err := file.ToTrimString(cfg) if err != nil { - log.Fatalln("read config file", cfg, "error:", err.Error()) + log.Fatal("read config file", cfg, "error:", err.Error()) } var c GlobalConfig err = json.Unmarshal([]byte(configContent), &c) if err != nil { - log.Fatalln("parse config file", cfg, "error:", err.Error()) + log.Fatal("parse config file", cfg, "error:", err.Error()) } if c.Migrate.Enabled && len(c.Migrate.Cluster) == 0 { @@ -112,5 +112,5 @@ func ParseConfig(cfg string) { // set config atomic.StorePointer(&ptr, unsafe.Pointer(&c)) - log.Println("g.ParseConfig ok, file", cfg) + log.Info("g.ParseConfig ok, file", cfg) } diff --git a/modules/graph/g/db.go b/modules/graph/g/db.go index 8e42bd0f5..139cd0bc0 100644 --- a/modules/graph/g/db.go +++ b/modules/graph/g/db.go @@ -17,7 +17,7 @@ package g import ( "database/sql" _ "github.com/go-sql-driver/mysql" - "log" + log "github.com/sirupsen/logrus" "sync" ) @@ -33,11 +33,11 @@ func InitDB() { var err error DB, err = makeDbConn() if DB == nil || err != nil { - log.Fatalln("g.InitDB, get db conn fail", err) + log.Fatal("g.InitDB, get db conn fail", err) } dbConnMap = make(map[string]*sql.DB) - log.Println("g.InitDB ok") + log.Info("g.InitDB ok") } func GetDbConn(connName string) (c *sql.DB, e error) { diff --git a/modules/graph/g/g.go b/modules/graph/g/g.go index 4a5aa1ae6..f4f5a3df2 100644 --- a/modules/graph/g/g.go +++ b/modules/graph/g/g.go @@ -15,7 +15,6 @@ package g import ( - "log" "runtime" ) @@ -55,7 +54,6 @@ const ( DEFAULT_STEP = 60 //s MIN_STEP = 30 //s CLEAN_CACHE = 86400 //s the step that clean GraphItems/historyCache Cache - CACHE_DELAY = 1800 //s CACHE_TIME = 1800000 //ms FLUSH_DISK_STEP = 1000 //ms FLUSH_MIN_COUNT = 6 // flush counter to disk when its number of monitoring data greater than FLUSH_MIN_COUNT @@ -71,5 +69,4 @@ const ( func init() { runtime.GOMAXPROCS(runtime.NumCPU()) - log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) } diff --git a/modules/graph/http/common.go b/modules/graph/http/common.go index 132fd3e09..78d6195e7 100644 --- a/modules/graph/http/common.go +++ b/modules/graph/http/common.go @@ -16,10 +16,13 @@ package http import ( "github.com/open-falcon/falcon-plus/modules/graph/g" + "github.com/open-falcon/falcon-plus/modules/graph/rrdtool" "github.com/open-falcon/falcon-plus/modules/graph/store" "github.com/toolkits/file" + "fmt" "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" "strconv" "time" ) @@ -73,4 +76,18 @@ func configCommonRoutes() { } JSONR(c, 200, rt) }) + + router.GET("/api/v2/counter/migrate", func(c *gin.Context) { + counter := rrdtool.GetCounterV2() + log.Debug("migrating counter v2:", fmt.Sprintf("%+v", counter)) + c.JSON(200, counter) + }) + + //compatible with open-falcon v0.1 + router.GET("/counter/migrate", func(c *gin.Context) { + cnt := rrdtool.GetCounter() + log.Debug("migrating counter:", cnt) + c.JSON(200, gin.H{"msg": "ok", "counter": cnt}) + }) + } diff --git a/modules/graph/http/http.go b/modules/graph/http/http.go index c987836f8..1e1cf8384 100644 --- a/modules/graph/http/http.go +++ b/modules/graph/http/http.go @@ -16,7 +16,6 @@ package http import ( "encoding/json" - "fmt" log "github.com/sirupsen/logrus" "net" "net/http" @@ -25,7 +24,6 @@ import ( "github.com/gin-gonic/gin" "github.com/open-falcon/falcon-plus/modules/graph/g" - "github.com/open-falcon/falcon-plus/modules/graph/rrdtool" ) type Dto struct { @@ -102,19 +100,6 @@ func Start() { configProcRoutes() configIndexRoutes() - router.GET("/api/v2/counter/migrate", func(c *gin.Context) { - counter := rrdtool.GetCounterV2() - log.Debug("migrating counter v2:", fmt.Sprintf("%+v", counter)) - c.JSON(200, counter) - }) - - //compatible with open-falcon v0.1 - router.GET("/counter/migrate", func(c *gin.Context) { - cnt := rrdtool.GetCounter() - log.Debug("migrating counter:", cnt) - c.JSON(200, gin.H{"msg": "ok", "counter": cnt}) - }) - addr := g.Config().Http.Listen if addr == "" { return diff --git a/modules/graph/index/cache.go b/modules/graph/index/cache.go index ac1da25f0..d32dbfbdd 100644 --- a/modules/graph/index/cache.go +++ b/modules/graph/index/cache.go @@ -17,7 +17,7 @@ package index import ( "database/sql" "fmt" - "log" + log "github.com/sirupsen/logrus" "strconv" "strings" "sync" @@ -97,7 +97,7 @@ func GetEndpointFromCache(endpoint string) (int64, bool) { var id int64 = -1 err := g.DB.QueryRow("SELECT id FROM endpoint WHERE endpoint = ?", endpoint).Scan(&id) if err != nil && err != sql.ErrNoRows { - log.Println("query endpoint id fail,", err) + log.Error("query endpoint id fail,", err) return -1, false } @@ -132,7 +132,7 @@ func GetCounterFromCache(endpointId int64, counter string) (dsType string, step err = g.DB.QueryRow("SELECT type, step FROM endpoint_counter WHERE endpoint_id = ? and counter = ?", endpointId, counter).Scan(&dsType, &step) if err != nil && err != sql.ErrNoRows { - log.Println("query type and step fail", err) + log.Error("query type and step fail", err) return } diff --git a/modules/graph/main.go b/modules/graph/main.go index 4467aa76a..5a9ebadd7 100644 --- a/modules/graph/main.go +++ b/modules/graph/main.go @@ -17,7 +17,7 @@ package main import ( "flag" "fmt" - "log" + log "github.com/sirupsen/logrus" "os" "os/signal" "syscall" @@ -32,33 +32,34 @@ import ( func start_signal(pid int, cfg *g.GlobalConfig) { sigs := make(chan os.Signal, 1) - log.Println(pid, "register signal notify") + log.Info(pid, " register signal notify") signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) for { s := <-sigs - log.Println("recv", s) + log.Info("recv", s) switch s { case syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT: - log.Println("graceful shut down") + log.Info("graceful shut down") if cfg.Http.Enabled { http.Close_chan <- 1 <-http.Close_done_chan } - log.Println("http stop ok") + log.Info("http stop ok") if cfg.Rpc.Enabled { api.Close_chan <- 1 <-api.Close_done_chan } - log.Println("rpc stop ok") + log.Info("rpc stop ok") - rrdtool.Out_done_chan <- 1 - rrdtool.FlushAll(true) - log.Println("rrdtool stop ok") + rrdtool.Main_done_chan <- 1 + //flush cache to local file or transmit cache to remote graph + rrdtool.CommitBeforeQuit() + log.Info("rrdtool stop ok") - log.Println(pid, "exit") + log.Info("pid:", pid, " exit") os.Exit(0) } } diff --git a/modules/graph/rrdtool/migrate.go b/modules/graph/rrdtool/migrate.go index 56441f748..9a2f52b2e 100644 --- a/modules/graph/rrdtool/migrate.go +++ b/modules/graph/rrdtool/migrate.go @@ -17,10 +17,10 @@ package rrdtool import ( "errors" "fmt" - "log" + log "github.com/sirupsen/logrus" "net" "net/rpc" - "os" + "strings" "sync/atomic" "time" @@ -144,7 +144,7 @@ func migrate_start(cfg *g.GlobalConfig) { for i = 0; i < cfg.Migrate.Concurrency; i++ { if clients[node][i], err = dial(addr, time.Second); err != nil { - log.Fatalf("node:%s addr:%s err:%s\n", node, addr, err) + log.Fatalf("node:%s addr:%s err:%s", node, addr, err) } go net_task_worker(i, Net_task_ch[node], &clients[node][i], addr) } @@ -174,32 +174,25 @@ func net_task_worker(idx int, ch chan *Net_task_t, client **rpc.Client, addr str atomic.AddUint64(&stat_cnt[QUERY_S_SUCCESS], 1) } } else if task.Method == NET_TASK_M_PULL { - if atomic.LoadInt32(&flushrrd_timeout) != 0 { - // hope this more faster than fetch_rrd - if err = send_data(client, task.Key, addr); err != nil { - pfc.Meter("migrate.sendbusy.err", 1) - atomic.AddUint64(&stat_cnt[SEND_S_ERR], 1) + if err = fetch_rrd(client, task.Key, addr); err != nil { + if strings.HasSuffix(err.Error(), "no such file or directory") { + // which expect that err msg like open xxx.rrd: no such file or directory + // TODO:check error in cross diffrent platforms + pfc.Meter("migrate.scprrd.null", 1) + atomic.AddUint64(&stat_cnt[FETCH_S_ISNOTEXIST], 1) + store.GraphItems.SetFlag(task.Key, 0) + //when the remote rrd file not exist, flush cache to local rrdfile + CommitByKey(task.Key) } else { - pfc.Meter("migrate.sendbusy.ok", 1) - atomic.AddUint64(&stat_cnt[SEND_S_SUCCESS], 1) + pfc.Meter("migrate.scprrd.err", 1) + //warning:other errors, cache backlogged + atomic.AddUint64(&stat_cnt[FETCH_S_ERR], 1) } } else { - if err = fetch_rrd(client, task.Key, addr); err != nil { - if os.IsNotExist(err) { - pfc.Meter("migrate.scprrd.null", 1) - //文件不存在时，直接将缓存数据刷入本地 - atomic.AddUint64(&stat_cnt[FETCH_S_ISNOTEXIST], 1) - store.GraphItems.SetFlag(task.Key, 0) - CommitByKey(task.Key) - } else { - pfc.Meter("migrate.scprrd.err", 1) - //warning:其他异常情况，缓存数据会堆积 - atomic.AddUint64(&stat_cnt[FETCH_S_ERR], 1) - } - } else { - pfc.Meter("migrate.scprrd.ok", 1) - atomic.AddUint64(&stat_cnt[FETCH_S_SUCCESS], 1) - } + pfc.Meter("migrate.scprrd.ok", 1) + atomic.AddUint64(&stat_cnt[FETCH_S_SUCCESS], 1) + //flush cache to local rrd file after scp rrd success + CommitByKey(task.Key) } } else { err = errors.New("error net task method") @@ -287,9 +280,11 @@ func send_data(client **rpc.Client, key string, addr string) error { if err == rpc.ErrShutdown { reconnection(client, addr) } + log.Errorf("transmit rrd %s %s to remote succ, retry[%d]", key, items[0].UUID(), i) } - // err - store.GraphItems.PushAll(key, items) + //send_data only will be called when graph quit, + //so then there is a error of transmit, drop the cache instead of restore. + //store.GraphItems.PushAll(key, items) //flag |= g.GRAPH_F_ERR out: flag &= ^g.GRAPH_F_SENDING @@ -341,7 +336,7 @@ func fetch_rrd(client **rpc.Client, key string, addr string) error { goto out } } else { - log.Println(err) + log.Errorf("scp rrd %s from remote err[%s], retry[%d]", key, err, i) } if err == rpc.ErrShutdown { reconnection(client, addr) diff --git a/modules/graph/rrdtool/rrdtool.go b/modules/graph/rrdtool/rrdtool.go index 721bd7f1a..536b02994 100644 --- a/modules/graph/rrdtool/rrdtool.go +++ b/modules/graph/rrdtool/rrdtool.go @@ -16,7 +16,7 @@ package rrdtool import ( "errors" - "log" + log "github.com/sirupsen/logrus" "math" "sync/atomic" "time" @@ -58,15 +58,17 @@ func Start() { var err error // check data dir if err = file.EnsureDirRW(cfg.RRD.Storage); err != nil { - log.Fatalln("rrdtool.Start error, bad data dir "+cfg.RRD.Storage+",", err) + log.Fatal("rrdtool.Start error, bad data dir "+cfg.RRD.Storage+",", err) } migrate_start(cfg) + log.Info("rrdtool migrateWorker started") - // sync disk go syncDisk() + log.Info("rrdtool syncDiskWorker started") + go ioWorker() - log.Println("rrdtool.Start ok") + log.Info("rrdtool ioWorker started") } // RRA.Point.Size @@ -169,7 +171,7 @@ func ReadFile(filename, md5 string) ([]byte, error) { return task.args.(*readfile_t).data, err } -func FlushFile(filename, md5 string, items []*cmodel.GraphItem) error { +func CommitFile(filename, md5 string, items []*cmodel.GraphItem) error { done := make(chan error, 1) io_task_chans[getIndex(md5)] <- &io_task_t{ method: IO_TASK_M_FLUSH, @@ -232,30 +234,18 @@ func fetch(filename string, cf string, start, end int64, step int) ([]*cmodel.RR return ret, nil } -func FlushAll(force bool) { - n := store.GraphItems.Size / 10 - for i := 0; i < store.GraphItems.Size; i++ { - FlushRRD(i, force) - if i%n == 0 { - log.Printf("flush hash idx:%03d size:%03d disk:%08d net:%08d\n", - i, store.GraphItems.Size, disk_counter, net_counter) - } - } - log.Printf("flush hash done (disk:%08d net:%08d)\n", disk_counter, net_counter) -} - -func CommitByKey(key string) { +func CommitByKey(key string) error { md5, dsType, step, err := g.SplitRrdCacheKey(key) if err != nil { - return + return err } filename := g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step) items := store.GraphItems.PopAll(key) if len(items) == 0 { - return + return nil } - FlushFile(filename, md5, items) + return CommitFile(filename, md5, items) } func PullByKey(key string) { @@ -274,45 +264,123 @@ func PullByKey(key string) { Key: key, Done: done, } - // net_task slow, shouldn't block syncDisk() or FlushAll() + // net_task slow, shouldn't block syncDisk() or CommitBeforeQuit() // warning: recev sigout when migrating, maybe lost memory data go func() { err := <-done if err != nil { - log.Printf("get %s from remote err[%s]\n", key, err) + log.Errorf("get %s %s from remote err[%s]", key, item.UUID(), err) return } atomic.AddUint64(&net_counter, 1) - //todo: flushfile after getfile? not yet }() } -func FlushRRD(idx int, force bool) { +func SendByKey(key string) { + done := make(chan error) + + item := store.GraphItems.First(key) + if item == nil { + return + } + node, err := Consistent.Get(item.PrimaryKey()) + if err != nil { + return + } + Net_task_ch[node] <- &Net_task_t{ + Method: NET_TASK_M_SEND, + Key: key, + Done: done, + } + + go func() { + err := <-done + if err != nil { + log.Errorf("transmit %s %s to remote err[%s]", key, item.UUID(), err) + } else { + log.Debugf("transmit %s %s to remote succ", key, item.UUID()) + } + }() +} + +func CommitBeforeQuit() { + n := store.GraphItems.Size / 10 + for i := 0; i < store.GraphItems.Size; i++ { + commitByIdxBeforeQuit(i) + if i%n == 0 { + log.Infof("flush rrd before quit, hash idx:%03d size:%03d disk:%08d net:%08d", + i, store.GraphItems.Size, disk_counter, net_counter) + } + } + log.Infof("flush done (disk:%08d net:%08d)", disk_counter, net_counter) +} + +func commitByIdxBeforeQuit(idx int) { begin := time.Now() - atomic.StoreInt32(&flushrrd_timeout, 0) keys := store.GraphItems.KeysByIndex(idx) if len(keys) == 0 { return } + is_migrate := g.Config().Migrate.Enabled for _, key := range keys { flag, _ := store.GraphItems.GetFlag(key) - //write err data to local filename - if force == false && g.Config().Migrate.Enabled && flag&g.GRAPH_F_MISS != 0 { - if time.Since(begin) > time.Millisecond*g.FLUSH_DISK_STEP { - atomic.StoreInt32(&flushrrd_timeout, 1) + if is_migrate && flag&g.GRAPH_F_MISS != 0 { + filename, _ := getFilenameByKey(key) + if !g.IsRrdFileExist(filename) { + //transmit cache data to remote graph + SendByKey(key) + } else { + CommitByKey(key) } - PullByKey(key) - } else if force || shouldFlush(key) { + } else { CommitByKey(key) } + + //check if there is backlog + if time.Since(begin) > time.Millisecond*g.FLUSH_DISK_STEP { + log.Warnf("commit rrd too slow, check the backlog of idx %d", idx) + } } } -func shouldFlush(key string) bool { +func commitByIdx(idx int) { + begin := time.Now() + keys := store.GraphItems.KeysByIndex(idx) + if len(keys) == 0 { + return + } + + is_migrate := g.Config().Migrate.Enabled + for _, key := range keys { + flag, _ := store.GraphItems.GetFlag(key) + if is_migrate { + if flag&g.GRAPH_F_MISS == 0 && shouldFlush(key) { + CommitByKey(key) + } + if flag&g.GRAPH_F_MISS != 0 { + filename, _ := getFilenameByKey(key) + if !g.IsRrdFileExist(filename) { + PullByKey(key) + } else { + CommitByKey(key) + } + } + } else if shouldFlush(key) { + CommitByKey(key) + } + + //check if there is backlog + if time.Since(begin) > time.Millisecond*g.FLUSH_DISK_STEP { + log.Warnf("commit rrd too slow, check the backlog of idx %d", idx) + } + } +} + +func shouldFlush(key string) bool { if store.GraphItems.ItemCnt(key) >= g.FLUSH_MIN_COUNT { return true } @@ -325,3 +393,11 @@ func shouldFlush(key string) bool { return false } + +func getFilenameByKey(key string) (string, error) { + md5, dsType, step, err := g.SplitRrdCacheKey(key) + if err != nil { + return "", err + } + return g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step), nil +} diff --git a/modules/graph/rrdtool/sync_disk.go b/modules/graph/rrdtool/sync_disk.go index fc294e2ee..322a7ae67 100644 --- a/modules/graph/rrdtool/sync_disk.go +++ b/modules/graph/rrdtool/sync_disk.go @@ -17,12 +17,12 @@ package rrdtool import ( "io" "io/ioutil" - "log" "os" "time" "github.com/open-falcon/falcon-plus/modules/graph/g" "github.com/open-falcon/falcon-plus/modules/graph/store" + log "github.com/sirupsen/logrus" "github.com/toolkits/file" ) @@ -41,33 +41,38 @@ type io_task_t struct { } var ( - Out_done_chan chan int - io_task_chans []chan *io_task_t + Main_done_chan chan int + io_task_chans []chan *io_task_t ) func InitChannel() { - Out_done_chan = make(chan int, 1) + Main_done_chan = make(chan int, 1) ioWorkerNum := g.Config().IOWorkerNum io_task_chans = make([]chan *io_task_t, ioWorkerNum) for i := 0; i < ioWorkerNum; i++ { + //the io task queue length is 16 io_task_chans[i] = make(chan *io_task_t, 16) } } func syncDisk() { - time.Sleep(time.Second * g.CACHE_DELAY) ticker := time.NewTicker(time.Millisecond * g.FLUSH_DISK_STEP) defer ticker.Stop() var idx int = 0 + n := store.GraphItems.Size / 10 for { select { case <-ticker.C: idx = idx % store.GraphItems.Size - FlushRRD(idx, false) + commitByIdx(idx) + if idx%n == 0 { + log.Debugf("flush rrd hash idx:%03d size:%03d disk:%08d net:%08d", + idx, store.GraphItems.Size, disk_counter, net_counter) + } idx += 1 - case <-Out_done_chan: - log.Println("cron recv sigout and exit...") + case <-Main_done_chan: + log.Info("syncDisk cron recv sigout and exit...") return } } diff --git a/modules/graph/store/store.go b/modules/graph/store/store.go index 6ecf6d66c..12325ce31 100644 --- a/modules/graph/store/store.go +++ b/modules/graph/store/store.go @@ -17,8 +17,8 @@ package store import ( "container/list" "errors" + log "github.com/sirupsen/logrus" "hash/crc32" - "log" "sync" cmodel "github.com/open-falcon/falcon-plus/common/model" @@ -37,7 +37,7 @@ type GraphItemMap struct { func (this *GraphItemMap) Get(key string) (*SafeLinkedList, bool) { this.RLock() defer this.RUnlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) val, ok := this.A[idx][key] return val, ok } @@ -46,7 +46,7 @@ func (this *GraphItemMap) Get(key string) (*SafeLinkedList, bool) { func (this *GraphItemMap) Remove(key string) bool { this.Lock() defer this.Unlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) _, exists := this.A[idx][key] if !exists { return false @@ -67,7 +67,7 @@ func (this *GraphItemMap) Getitems(idx int) map[string]*SafeLinkedList { func (this *GraphItemMap) Set(key string, val *SafeLinkedList) { this.Lock() defer this.Unlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) this.A[idx][key] = val } @@ -84,7 +84,7 @@ func (this *GraphItemMap) Len() int { func (this *GraphItemMap) First(key string) *cmodel.GraphItem { this.RLock() defer this.RUnlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) sl, ok := this.A[idx][key] if !ok { return nil @@ -101,7 +101,7 @@ func (this *GraphItemMap) First(key string) *cmodel.GraphItem { func (this *GraphItemMap) PushAll(key string, items []*cmodel.GraphItem) error { this.Lock() defer this.Unlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) sl, ok := this.A[idx][key] if !ok { return errors.New("not exist") @@ -113,7 +113,7 @@ func (this *GraphItemMap) PushAll(key string, items []*cmodel.GraphItem) error { func (this *GraphItemMap) GetFlag(key string) (uint32, error) { this.Lock() defer this.Unlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) sl, ok := this.A[idx][key] if !ok { return 0, errors.New("not exist") @@ -121,10 +121,14 @@ func (this *GraphItemMap) GetFlag(key string) (uint32, error) { return sl.Flag, nil } +func (this *GraphItemMap) idx(key string) uint32 { + return hashKey(key) % uint32(this.Size) +} + func (this *GraphItemMap) SetFlag(key string, flag uint32) error { this.Lock() defer this.Unlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) sl, ok := this.A[idx][key] if !ok { return errors.New("not exist") @@ -136,7 +140,7 @@ func (this *GraphItemMap) SetFlag(key string, flag uint32) error { func (this *GraphItemMap) PopAll(key string) []*cmodel.GraphItem { this.Lock() defer this.Unlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) sl, ok := this.A[idx][key] if !ok { return []*cmodel.GraphItem{} @@ -147,7 +151,7 @@ func (this *GraphItemMap) PopAll(key string) []*cmodel.GraphItem { func (this *GraphItemMap) FetchAll(key string) ([]*cmodel.GraphItem, uint32) { this.RLock() defer this.RUnlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) sl, ok := this.A[idx][key] if !ok { return []*cmodel.GraphItem{}, 0 @@ -175,7 +179,6 @@ func (this *GraphItemMap) PushFront(key string, if linkedList, exists := this.Get(key); exists { linkedList.PushFront(item) } else { - //log.Println("new key:", key) safeList := &SafeLinkedList{L: list.New()} safeList.L.PushFront(item) @@ -207,7 +210,7 @@ func (this *GraphItemMap) KeysByIndex(idx int) []string { func (this *GraphItemMap) Back(key string) *cmodel.GraphItem { this.RLock() defer this.RUnlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) L, ok := this.A[idx][key] if !ok { return nil @@ -225,7 +228,7 @@ func (this *GraphItemMap) Back(key string) *cmodel.GraphItem { func (this *GraphItemMap) ItemCnt(key string) int { this.RLock() defer this.RUnlock() - idx := hashKey(key) % uint32(this.Size) + idx := this.idx(key) L, ok := this.A[idx][key] if !ok { return 0 From ae99a756218a8cbf7a009493ec939e31f1e57f0a Mon Sep 17 00:00:00 2001 From: laiwei Date: Fri, 14 Feb 2020 22:31:48 +0800 Subject: [PATCH 09/10] update vendor rrdlite --- .../github.com/open-falcon/rrdlite/rrd_c.go | 23 +++++++++---------- .../open-falcon/rrdlite/rrd_error.h | 2 +- .../github.com/open-falcon/rrdlite/rrdfunc.c | 16 +++++++++++++ vendor/vendor.json | 6 ++--- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/vendor/github.com/open-falcon/rrdlite/rrd_c.go b/vendor/github.com/open-falcon/rrdlite/rrd_c.go index bea526528..d79641fb0 100644 --- a/vendor/github.com/open-falcon/rrdlite/rrd_c.go +++ b/vendor/github.com/open-falcon/rrdlite/rrd_c.go @@ -14,14 +14,11 @@ import ( "reflect" "strconv" "strings" - "sync" "time" "unsafe" ) -var mutex sync.Mutex - -func makeArgs(args []string) []*C.char { +func makeCArgs(args []string) []*C.char { ret := make([]*C.char, len(args)) for i, s := range args { ret[i] = C.CString(s) @@ -39,7 +36,7 @@ func freeArgs(cArgs []*C.char) { } } -func makeError(e *C.char) error { +func makeGoError(e *C.char) error { var null *C.char if e == null { return nil @@ -50,7 +47,7 @@ func makeError(e *C.char) error { func (c *Creator) create() error { filename := C.CString(c.filename) defer freeCString(filename) - args := makeArgs(c.args) + args := makeCArgs(c.args) defer freeArgs(args) e := C.rrdCreate( @@ -60,12 +57,12 @@ func (c *Creator) create() error { C.int(len(args)), &args[0], ) - return makeError(e) + return makeGoError(e) } func (u *Updater) update(_args []string) error { - args := makeArgs(_args) + args := makeCArgs(_args) defer freeArgs(args) e := C.rrdUpdate( @@ -74,7 +71,7 @@ func (u *Updater) update(_args []string) error { C.int(len(args)), &args[0], ) - return makeError(e) + return makeGoError(e) } var ( @@ -245,7 +242,7 @@ func Info(filename string) (map[string]interface{}, error) { fn := C.CString(filename) defer freeCString(fn) var i *C.rrd_info_t - err := makeError(C.rrdInfo(&i, fn)) + err := makeGoError(C.rrdInfo(&i, fn)) if err != nil { return nil, err } @@ -254,6 +251,7 @@ func Info(filename string) (map[string]interface{}, error) { // Fetch retrieves data from RRD file. func Fetch(filename, cf string, start, end time.Time, step time.Duration) (FetchResult, error) { + fn := C.CString(filename) defer freeCString(fn) cCf := C.CString(cf) @@ -262,12 +260,13 @@ func Fetch(filename, cf string, start, end time.Time, step time.Duration) (Fetch cEnd := C.time_t(end.Unix()) cStep := C.ulong(step.Seconds()) var ( - ret C.int + cRet C.int cDsCnt C.ulong cDsNames **C.char cData *C.double ) - err := makeError(C.rrdFetch(&ret, fn, cCf, &cStart, &cEnd, &cStep, &cDsCnt, &cDsNames, &cData)) + + err := makeGoError(C.rrdFetch(&cRet, fn, cCf, &cStart, &cEnd, &cStep, &cDsCnt, &cDsNames, &cData)) if err != nil { return FetchResult{filename, cf, start, end, step, nil, 0, nil}, err } diff --git a/vendor/github.com/open-falcon/rrdlite/rrd_error.h b/vendor/github.com/open-falcon/rrdlite/rrd_error.h index f55078cf0..aecb29f1b 100644 --- a/vendor/github.com/open-falcon/rrdlite/rrd_error.h +++ b/vendor/github.com/open-falcon/rrdlite/rrd_error.h @@ -123,4 +123,4 @@ #define RRD_ERR_UNKNOWN_DATA1 0x0279 /* if add new system event flag, please upadte the RRD_ERR_END */ #define RRD_ERR_END 0x0279 -#define RRD_ERR_NUM (RRD_ERR_END - RRD_ERR_START + 1) \ No newline at end of file +#define RRD_ERR_NUM (RRD_ERR_END - RRD_ERR_START + 1) diff --git a/vendor/github.com/open-falcon/rrdlite/rrdfunc.c b/vendor/github.com/open-falcon/rrdlite/rrdfunc.c index 62a21cbc5..13d460cd8 100644 --- a/vendor/github.com/open-falcon/rrdlite/rrdfunc.c +++ b/vendor/github.com/open-falcon/rrdlite/rrdfunc.c @@ -1,10 +1,15 @@ #include +#include + #include "rrd.h" const char *rrdCreate(const char *filename, unsigned long step, time_t start, int argc, const char **argv) { int ret; ret = rrd_create_r(filename, step, start, argc, argv); + if (ret != 0){ + printf("rrd_create %s, errno: %d\n", filename, ret); + } return rrd_strerror(ret); } @@ -12,6 +17,9 @@ const char *rrdUpdate(const char *filename, const char *template, int argc, const char **argv) { int ret; ret = rrd_update_r(filename, template, argc, argv); + if (ret != 0){ + printf("rrd_update %s, errno: %d\n", filename, ret); + } return rrd_strerror(ret); } @@ -19,13 +27,21 @@ const char *rrdUpdate(const char *filename, const char *template, const char *rrdInfo(rrd_info_t **info, char *filename) { int ret = 0; *info = rrd_info_r(filename, &ret); + if (ret != 0){ + printf("rrd_info %s, errno: %d\n", filename, ret); + } return rrd_strerror(ret); } const char *rrdFetch(int *ret, char *filename, const char *cf, time_t *start, time_t *end, unsigned long *step, unsigned long *ds_cnt, char ***ds_namv, double **data) { + //setbuf(stdout, NULL); + //setbuf(stderr, NULL); *ret = rrd_fetch_r(filename, cf, start, end, step, ds_cnt, ds_namv, data); + if (*ret != 0){ + printf("rrdfetch %s, errno: %d\n", filename, *ret); + } return rrd_strerror(*ret); } diff --git a/vendor/vendor.json b/vendor/vendor.json index 1122e8355..a0309884b 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -307,10 +307,10 @@ "revisionTime": "2015-12-21T08:53:10Z" }, { - "checksumSHA1": "I0kIkGOtYBit3XflqEqbEt+kuhQ=", + "checksumSHA1": "8tcXzsjEx8Qo/4wv0+T0/CZJqqk=", "path": "github.com/open-falcon/rrdlite", - "revision": "7d8646c85cc56cad1f65d97abbe24fdfc1b88917", - "revisionTime": "2017-04-12T12:20:36Z" + "revision": "bf5829f786ad3765b09ae54f416106d5f1565c07", + "revisionTime": "2020-02-14T14:08:04Z" }, { "checksumSHA1": "F1IYMLBLAZaTOWnmXsgaxTGvrWI=", From cc1922a89e9581580a67fe1667d5d9bea4b95192 Mon Sep 17 00:00:00 2001 From: laiwei Date: Fri, 10 Apr 2020 23:48:24 +0800 Subject: [PATCH 10/10] fix typo --- modules/graph/rrdtool/migrate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/graph/rrdtool/migrate.go b/modules/graph/rrdtool/migrate.go index 9a2f52b2e..f97c84b70 100644 --- a/modules/graph/rrdtool/migrate.go +++ b/modules/graph/rrdtool/migrate.go @@ -177,7 +177,7 @@ func net_task_worker(idx int, ch chan *Net_task_t, client **rpc.Client, addr str if err = fetch_rrd(client, task.Key, addr); err != nil { if strings.HasSuffix(err.Error(), "no such file or directory") { // which expect that err msg like open xxx.rrd: no such file or directory - // TODO:check error in cross diffrent platforms + // TODO:check error in cross different platforms pfc.Meter("migrate.scprrd.null", 1) atomic.AddUint64(&stat_cnt[FETCH_S_ISNOTEXIST], 1) store.GraphItems.SetFlag(task.Key, 0)