From 8dbb9de61a1fdf2ce93b426e1b8700ad32b4127c Mon Sep 17 00:00:00 2001 From: Sergey Chubaryan Date: Sat, 17 Aug 2024 15:09:29 +0300 Subject: [PATCH] load tests with Locust/Python, fixes --- .gitignore | 7 +- .vscode/launch.json | 16 -- go.mod | 2 +- go.sum | 2 + load_tests/.gitignore | 162 +++++++++++++++++++++ load_tests/locustfile.py | 54 +++++++ load_tests/makefile | 15 ++ load_tests/requirements.txt | 27 ++++ main.go | 64 +++++++- makefile | 11 +- misc/config.yaml | 3 +- src/args_parser/args.go | 16 +- src/server/handlers/user_create_handler.go | 6 +- src/server/handlers/user_login_handler.go | 4 +- 14 files changed, 352 insertions(+), 37 deletions(-) delete mode 100644 .vscode/launch.json create mode 100644 load_tests/.gitignore create mode 100644 load_tests/locustfile.py create mode 100644 load_tests/makefile create mode 100644 load_tests/requirements.txt diff --git a/.gitignore b/.gitignore index fec6d12..0573c33 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore -# + +# Ide +.vscode/* + # Binaries for programs and plugins *.exe *.exe~ @@ -26,7 +29,7 @@ go.work.sum # env file .env - +.pyenv .run # temporary diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 797be4e..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Launch Package", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}", - "args": ["-c", "./config_example/config.yaml", "-o", "./log.txt"] - } - ] -} \ No newline at end of file diff --git a/go.mod b/go.mod index 39a6425..81693bf 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/net v0.27.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sys v0.24.0 // indirect golang.org/x/text v0.16.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect diff --git a/go.sum b/go.sum index 2dbc55c..c9d0678 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/load_tests/.gitignore b/load_tests/.gitignore new file mode 100644 index 0000000..efa407c --- /dev/null +++ b/load_tests/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/load_tests/locustfile.py b/load_tests/locustfile.py new file mode 100644 index 0000000..a608d57 --- /dev/null +++ b/load_tests/locustfile.py @@ -0,0 +1,54 @@ +import random +import string +from locust import HttpUser, task + +class DummyRoute(HttpUser): + @task + def dummy_test(self): + self.client.get("/dummy") + + # @task + # def user_create_test(self): + # randEmail = ''.join(random.choices(string.ascii_lowercase + string.digits, k=10)) + '@test.test' + # randPassword = ''.join(random.choices(string.ascii_letters + string.digits, k=10)) + # randName = ''.join(random.choices(string.ascii_letters, k=10)) + # self.client.post( + # "/user/create", + # json={ + # "email":randEmail, + # "password":randPassword, + # "name": randName, + # }, + # ) + + def on_start(self): + randEmail = ''.join(random.choices(string.ascii_lowercase + string.digits, k=10)) + '@test.test' + randPassword = ''.join(random.choices(string.ascii_letters + string.digits, k=10)) + randName = ''.join(random.choices(string.ascii_letters, k=10)) + + response = self.client.post( + "/user/create", + json={ + "email":randEmail, + "password":randPassword, + "name": randName, + }, + ) + if response.status_code != 200: + raise AssertionError('can not create user') + + response = self.client.post( + "/user/login", + json={ + "email":randEmail, + "password":randPassword, + }, + ) + if response.status_code != 200: + raise AssertionError('can not login user') + + token = response.json()['token'] + if token == '': + raise AssertionError('empty user token') + + self.client.headers = {"X-Auth": token} \ No newline at end of file diff --git a/load_tests/makefile b/load_tests/makefile new file mode 100644 index 0000000..033153a --- /dev/null +++ b/load_tests/makefile @@ -0,0 +1,15 @@ +SHELL := /bin/bash + +all: venv install + +venv: + python3 -m pip install virtualenv + virtualenv .venv + # source .venv/bin/activate + +install: + source .venv/bin/activate + python3 -m pip install -r requirements.txt + +run-web: + locust --host http://localhost:8080 \ No newline at end of file diff --git a/load_tests/requirements.txt b/load_tests/requirements.txt new file mode 100644 index 0000000..166fda7 --- /dev/null +++ b/load_tests/requirements.txt @@ -0,0 +1,27 @@ +blinker==1.8.2 +Brotli==1.1.0 +certifi==2024.7.4 +charset-normalizer==3.3.2 +click==8.1.7 +ConfigArgParse==1.7 +Flask==3.0.3 +Flask-Cors==4.0.1 +Flask-Login==0.6.3 +gevent==24.2.1 +geventhttpclient==2.3.1 +greenlet==3.0.3 +idna==3.7 +itsdangerous==2.2.0 +Jinja2==3.1.4 +locust==2.31.3 +MarkupSafe==2.1.5 +msgpack==1.0.8 +psutil==6.0.0 +pyzmq==26.1.0 +requests==2.32.3 +tomli==2.0.1 +typing_extensions==4.12.2 +urllib3==2.2.2 +Werkzeug==3.0.3 +zope.event==5.0 +zope.interface==7.0.1 diff --git a/main.go b/main.go index 40ac6e1..921d4f4 100644 --- a/main.go +++ b/main.go @@ -11,12 +11,17 @@ import ( "backend/src/logger" "backend/src/server/handlers" "backend/src/server/middleware" + "context" "crypto/rsa" "crypto/x509" "database/sql" "encoding/pem" "fmt" + "net" "os" + "os/signal" + "runtime/pprof" + "syscall" "github.com/gin-gonic/gin" "github.com/jackc/pgx" @@ -40,6 +45,22 @@ func main() { } logger.Log().Msg("initializing service...") + defer logger.Log().Msg("service stopped") + + signals := []os.Signal{ + os.Kill, + os.Interrupt, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGKILL, + syscall.SIGSTOP, + syscall.SIGQUIT, + syscall.SIGABRT, + syscall.SIGHUP, + } + + ctx, stop := signal.NotifyContext(context.Background(), signals...) + defer stop() conf, err := config.NewFromFile(args.GetConfigPath()) if err != nil { @@ -99,14 +120,16 @@ func main() { }, ) - if !debugMode { - gin.SetMode(gin.ReleaseMode) - } + // if !debugMode { + gin.SetMode(gin.ReleaseMode) + // } r := gin.New() r.Use(middleware.NewRequestLogMiddleware(logger)) r.Use(gin.Recovery()) + r.Static("/webapp", "./webapp") + linkGroup := r.Group("/s") linkGroup.POST("/new", handlers.NewShortlinkCreateHandler(linkService)) linkGroup.GET("/:linkId", handlers.NewShortlinkResolveHandler(linkService)) @@ -116,17 +139,46 @@ func main() { userGroup.POST("/login", handlers.NewUserLoginHandler(userService)) dummyGroup := r.Group("/dummy") - dummyGroup.Use(middleware.NewAuthMiddleware(userService)) - dummyGroup.GET("/", handlers.NewDummyHandler()) + { + dummyGroup.Use(middleware.NewAuthMiddleware(userService)) + dummyGroup.GET("", handlers.NewDummyHandler()) + } lpGroup := r.Group("/pooling") lpGroup.GET("/", handlers.NewLongPoolingHandler(clientNotifier)) + if args.GetProfilePath() != "" { + pprofFile, err := os.Create(args.GetProfilePath()) + if err != nil { + logger.Fatal().Err(err).Msg("can not create profile file") + } + + if err := pprof.StartCPUProfile(pprofFile); err != nil { + logger.Fatal().Err(err).Msg("can not start cpu profiling") + } + defer func() { + logger.Log().Msg("stopping profiling...") + pprof.StopCPUProfile() + pprofFile.Close() + }() + } + listenAddr := fmt.Sprintf(":%d", conf.GetPort()) logger.Log().Msgf("server listening on %s", listenAddr) - err = r.Run(listenAddr) + listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", listenAddr) if err != nil { + logger.Fatal().Err(err).Msg("can not create network listener") + } + + go func() { + <-ctx.Done() + logger.Log().Msg("stopping service...") + listener.Close() + }() + + err = r.RunListener(listener) + if err != nil && err == net.ErrClosed { logger.Fatal().Err(err).Msg("server stopped with error") } } diff --git a/makefile b/makefile index 0b351d6..90e2374 100644 --- a/makefile +++ b/makefile @@ -1,4 +1,13 @@ all: release release: - go build -o ./.build/release/backend main.go \ No newline at end of file + GOEXPERIMENT=boringcrypto go build -ldflags "-s -w" -o ./.build/release/backend main.go + +run: release + ./.build/release/backend -c ./misc/config.yaml -o ./.run/log.txt -p ./.run/cpu.pprof + +venv: + python3 -m pip install --user virtualenv + +locust: + pip3 install locust \ No newline at end of file diff --git a/misc/config.yaml b/misc/config.yaml index c9b7276..4d101a5 100644 --- a/misc/config.yaml +++ b/misc/config.yaml @@ -1,4 +1,3 @@ port: 8080 postgres_url: "postgres://postgres:postgres@localhost:5432/postgres" -jwt_signing_key: "./config_example/jwt_signing_key" -log_file: "./log.txt" \ No newline at end of file +jwt_signing_key: "./misc/jwt_signing_key" \ No newline at end of file diff --git a/src/args_parser/args.go b/src/args_parser/args.go index 53df19f..543d0df 100644 --- a/src/args_parser/args.go +++ b/src/args_parser/args.go @@ -5,6 +5,7 @@ import ( ) type Args interface { + GetProfilePath() string GetConfigPath() string GetLogPath() string } @@ -14,6 +15,7 @@ func Parse(osArgs []string) (Args, error) { s := parser.String("c", "config", &argparse.Options{Required: true, Help: "Path to a config file"}) l := parser.String("o", "log", &argparse.Options{Required: false, Default: "", Help: "Path to a log file"}) + p := parser.String("p", "profile", &argparse.Options{Required: false, Default: "", Help: "Path to a cpu profile file"}) err := parser.Parse(osArgs) if err != nil { @@ -21,14 +23,16 @@ func Parse(osArgs []string) (Args, error) { } return &args{ - ConfigPath: *s, - LogPath: *l, + ConfigPath: *s, + LogPath: *l, + ProfilePath: *p, }, nil } type args struct { - ConfigPath string - LogPath string + ProfilePath string + ConfigPath string + LogPath string } func (a *args) GetConfigPath() string { @@ -38,3 +42,7 @@ func (a *args) GetConfigPath() string { func (a *args) GetLogPath() string { return a.LogPath } + +func (a *args) GetProfilePath() string { + return a.ProfilePath +} diff --git a/src/server/handlers/user_create_handler.go b/src/server/handlers/user_create_handler.go index 28ad72c..9591a13 100644 --- a/src/server/handlers/user_create_handler.go +++ b/src/server/handlers/user_create_handler.go @@ -8,9 +8,9 @@ import ( ) type createUserInput struct { - Email string - Password string - Name string + Email string `json:"email"` + Password string `json:"password"` + Name string `json:"name"` } type createUserOutput struct { diff --git a/src/server/handlers/user_login_handler.go b/src/server/handlers/user_login_handler.go index 00eec5d..4250728 100644 --- a/src/server/handlers/user_login_handler.go +++ b/src/server/handlers/user_login_handler.go @@ -8,8 +8,8 @@ import ( ) type loginUserInput struct { - Login string - Password string + Login string `json:"email"` + Password string `json:"password"` } type loginUserOutput struct {