diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5ca4b65 --- /dev/null +++ b/Makefile @@ -0,0 +1,204 @@ +GO ?= go +GOFMT ?= gofmt "-s" +GO_VERSION=$(shell $(GO) version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f2) +PACKAGES ?= $(shell $(GO) list ./...) +VETPACKAGES ?= $(shell $(GO) list ./...) +GOFILES := $(shell find . -name "*.go") +CORE_DIR := ./core +UI_DIR := ./ui +DOCS_DIR := ./docs +LIBRARIES_DIR := ./../libraries +PG_VERSION := 16.4-alpine3.20 +DB_NAME := rating +MOD_NAME := rating-orama + +.PHONY: sayhello +# Print Hello World +sayhello: + @echo "Hello World" + +.PHONY: dockerize +# Creates a development database. +dockerize: + docker run --name $(DB_NAME)-db-dev -e POSTGRES_PASSWORD=secret -e POSTGRES_USER=developer -e POSTGRES_DB=$(DB_NAME) -p 5432:5432 -d postgres:$(PG_VERSION) + +.PHONY: dockerize-test +# Creates a test database. +dockerize-test: + docker run --name $(DB_NAME)-db-test -e POSTGRES_PASSWORD=secret -e POSTGRES_USER=developer -e POSTGRES_DB=$(DB_NAME) -p 5433:5432 -d postgres:$(PG_VERSION) + +.PHONY: undockerize +# Destroy a development database. +undockerize: + docker rm -f $(DB_NAME)-db-dev + +.PHONY: undockerize-test +# Destroy a test database. +undockerize-test: + docker rm -f $(DB_NAME)-db-test + +.PHONY: restart-db +# Restart a development database. +restart-db: + make undockerize + make dockerize + +.PHONY: restart-db-test +# Restart a test database. +restart-db-test: + make undockerize-test + make dockerize-test + +.PHONY: migrateup +# Migrate all schemas, triggers and data located in database/migrations. +migrateup: + migrate -path $(CORE_DIR)/cmd/database/migrations -database "postgresql://developer:secret@localhost:5432/$(DB_NAME)?sslmode=disable" -verbose up + +.PHONY: migratedown +# Migrate all schemas, triggers and data located in database/migrations. +migratedown: + migrate -path $(CORE_DIR)/cmd/database/migrations -database "postgresql://developer:secret@localhost:5432/$(DB_NAME)?sslmode=disable" -verbose down + +.PHONY: pg-dump +# Dump database to file. +pg-dump: + docker exec -e PGPASSWORD=secret $(DB_NAME)-db-dev pg_dump -U developer --column-inserts --data-only $(DB_NAME) > $(CORE_DIR)/cmd/database/data/data.sql + sed -i '1iSET session_replication_role = '\''replica'\'';' $(CORE_DIR)/cmd/database/data/data.sql + sed -i '$$aSET session_replication_role = '\''origin'\'';' $(CORE_DIR)/cmd/database/data/data.sql + +.PHONY: pg-restore +# Restore database from file. +pg-restore: + docker cp $(CORE_DIR)/cmd/database/data/data.sql $(DB_NAME)-db-dev:/data.sql + docker exec -e PGPASSWORD=secret $(DB_NAME)-db-dev psql -U developer -d $(DB_NAME) -f data.sql + +.PHONY: pg-docs +# Generate docs from database. +pg-docs: + java -jar $(LIBRARIES_DIR)/schemaspy-6.2.4.jar -t pgsql -dp $(LIBRARIES_DIR)/postgresql-42.7.4.jar -db $(DB_NAME) -host localhost -port 5432 -u developer -p secret -o $(DOCS_DIR)/database -vizjs + +.PHONY: sqlc +# Generate or recreate SQLC queries. +sqlc: + cd $(CORE_DIR) && sqlc generate + make gomock + + +.PHONY: test +# Test all files and generate coverage file. +test: + cd $(CORE_DIR) && $(GO) test ./... -v -covermode=count -coverprofile=./benchmark/coverage.out $(PACKAGES) + +.PHONY: gomock +# Generate mock files. +gomock: + cd $(CORE_DIR) && mockgen -package mock -destination internal/repository/mock/querier.go $(MOD_NAME)/internal/repository ExtendedQuerier + +.PHONY: run +# Run project. +run: + cd $(CORE_DIR) && $(GO) run ./cmd/. + +.PHONY: bench +# Run benchmarks. +bench: + cd $(CORE_DIR) && test -f benchmark/new_benchmark.txt && mv benchmark/new_benchmark.txt benchmark/old_benchmark.txt || true + cd $(CORE_DIR) && $(GO) test ./... -bench=. -count=10 -benchmem > benchmark/new_benchmark.txt + cd $(CORE_DIR) && benchstat benchmark/old_benchmark.txt benchmark/new_benchmark.txt > benchmark/benchstat.txt + +.PHONY: recreate +# Destroy development DB and generate ones. +recreate: + echo "y" | make migratedown + make migrateup + +.PHONY: tidy +# Runs a go mod tidy +tidy: + cd $(CORE_DIR) && $(GO) mod tidy + +.PHONY: build-linux +# Build and generate linux executable. +build-linux: + cd $(CORE_DIR) && make tidy + cd $(CORE_DIR) && make remove-debug + cd $(CORE_DIR) && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./tmp/arena ./cmd/. + +.PHONY: pack-docker +# Run docker build for pack binary and assets to Docker container. +pack-docker: + make test + make build-linux + docker build -t $(MOD_NAME):${version} -t $(MOD_NAME):latest . + +.PHONY: remove-debug +# Remove all debug entries for reduce size binary. +remove-debug: + cd $(CORE_DIR) && find . -name "*.go" -type f -exec sed -i '/slog\.Debug/d' {} + + +.PHONY: fmt +# Ensure consistent code formatting. +fmt: + cd $(CORE_DIR) && $(GOFMT) -w $(GOFILES) + +.PHONY: fmt-check +# format (check only). +fmt-check: + @diff=$$($(GOFMT) -d $(GOFILES)); \ + if [ -n "$$diff" ]; then \ + echo "Please run 'make fmt' and commit the result:"; \ + echo "$${diff}"; \ + exit 1; \ + fi; + +.PHONY: vet +# Examine packages and report suspicious constructs if any. +vet: + cd $(CORE_DIR) && $(GO) vet $(VETPACKAGES) + +.PHONY: tools +# Install tools (migrate and sqlc). +tools: + @if [ $(GO_VERSION) -gt 16 ]; then \ + cd $(CORE_DIR) && $(GO) install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest; \ + cd $(CORE_DIR) && $(GO) install github.com/sqlc-dev/sqlc/cmd/sqlc@latest; \ + fi + +.PHONY: env +# Copy .env.example to .env if .env does not already exist +env: + cd $(CORE_DIR) && @if [ ! -f .env ]; then \ + cp .env.example .env; \ + echo ".env file created from .env.example"; \ + else \ + echo ".env file already exists"; \ + fi + + +.PHONY: first-run +# Runs for the first time +first-run: + make tools + make env + make recreate + make run + +.PHONY: help +# Help. +help: + @echo '' + @echo 'Usage:' + @echo ' make [target]' + @echo '' + @echo 'Targets:' + @awk '/^[a-zA-Z\-\0-9]+:/ { \ + helpMessage = match(lastLine, /^# (.*)/); \ + if (helpMessage) { \ + helpCommand = substr($$1, 0, index($$1, ":")-1); \ + helpMessage = substr(lastLine, RSTART + 2, RLENGTH); \ + printf " - \033[36m%-20s\033[0m %s\n", helpCommand, helpMessage; \ + } \ + } \ + { lastLine = $$0 }' $(MAKEFILE_LIST) + +.DEFAULT_GOAL := help \ No newline at end of file diff --git a/core/Makefile b/core/Makefile deleted file mode 100644 index 59a4c14..0000000 --- a/core/Makefile +++ /dev/null @@ -1,145 +0,0 @@ -GO ?= go -GOFMT ?= gofmt "-s" -GO_VERSION=$(shell $(GO) version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f2) -PACKAGES ?= $(shell $(GO) list ./...) -VETPACKAGES ?= $(shell $(GO) list ./...) -GOFILES := $(shell find . -name "*.go") - -.PHONY: sayhello -# Print Hello World -sayhello: - @echo "Hello World" - -.PHONY: dockerize -# Creates a development database. -dockerize: - docker run --name rating-db-dev -e POSTGRES_PASSWORD=secret -e POSTGRES_USER=developer -e POSTGRES_DB=rating -p 5432:5432 -d postgres:16.3-alpine3.20 - -.PHONY: undockerize -# Destroy a development database. -undockerize: - docker rm -f rating-db-dev - -.PHONY: migrateup -# Migrate all schemas, triggers and data located in cmd/database/migrations. -migrateup: - migrate -path cmd/database/migrations -database "postgresql://developer:secret@localhost:5432/rating?sslmode=disable" -verbose up - -.PHONY: sqlc -# Generate or recreate SQLC queries. -sqlc: - sqlc generate - -.PHONY: test -# Test all files and generate coverage file. -test: - $(GO) test -v -covermode=count -coverprofile=coverage.out $(PACKAGES) - -.PHONY: gomock -# Generate mock files. -gomock: - mockgen -package mock -destination internal/repository/mock/querier.go rating-orama/internal/repository ExtendedQuerier - -.PHONY: run -# Run project. -run: - $(GO) run ./cmd/. - -.PHONY: recreate -# Destroy development DB and generate ones. -recreate: - make undockerize - make dockerize - sleep 2 - make migrateup - -.PHONY: tidy -# Runs a go mod tidy -tidy: - $(GO) mod tidy - -.PHONY: build-linux -# Build and generate linux executable. -build-linux: - make tidy - make remove-debug - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./tmp/arena ./cmd/. - -.PHONY: pack-docker -# Run docker build for pack binary and assets to Docker container. -pack-docker: - make test - make build-linux - docker build -t rating-orama:${version} -t rating-orama:latest . - -.PHONY: remove-debug -# Remove all debug entries for reduce size binary. -remove-debug: - find . -name "*.go" -type f -exec sed -i '/slog\.Debug/d' {} + - -.PHONY: fmt -# Ensure consistent code formatting. -fmt: - $(GOFMT) -w $(GOFILES) - -.PHONY: fmt-check -# format (check only). -fmt-check: - @diff=$$($(GOFMT) -d $(GOFILES)); \ - if [ -n "$$diff" ]; then \ - echo "Please run 'make fmt' and commit the result:"; \ - echo "$${diff}"; \ - exit 1; \ - fi; - -.PHONY: vet -# Examine packages and report suspicious constructs if any. -vet: - $(GO) vet $(VETPACKAGES) - -.PHONY: tools -# Install tools (migrate and sqlc). -tools: - @if [ $(GO_VERSION) -gt 16 ]; then \ - $(GO) install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest; \ - $(GO) install github.com/sqlc-dev/sqlc/cmd/sqlc@latest; \ - fi - -.PHONY: env -# Copy .env.example to .env if .env does not already exist -env: - @if [ ! -f .env ]; then \ - cp .env.example .env; \ - echo ".env file created from .env.example"; \ - else \ - echo ".env file already exists"; \ - fi - - -.PHONY: first-run -# Runs for the first time -first-run: - make tools - make env - make recreate - make run - -.PHONY: help -# Help. -help: - @echo '' - @echo 'Usage:' - @echo ' make [target]' - @echo '' - @echo 'Targets:' - @awk '/^[a-zA-Z\-\0-9]+:/ { \ - helpMessage = match(lastLine, /^# (.*)/); \ - if (helpMessage) { \ - helpCommand = substr($$1, 0, index($$1, ":")-1); \ - helpMessage = substr(lastLine, RSTART + 2, RLENGTH); \ - printf " - \033[36m%-20s\033[0m %s\n", helpCommand, helpMessage; \ - } \ - } \ - { lastLine = $$0 }' $(MAKEFILE_LIST) - -.DEFAULT_GOAL := help \ No newline at end of file diff --git a/core/cmd/main.go b/core/cmd/main.go index 570613c..3ba4311 100644 --- a/core/cmd/main.go +++ b/core/cmd/main.go @@ -3,18 +3,15 @@ package main import ( "embed" "encoding/gob" - "gopher-toolbox/app" "gopher-toolbox/db" "log/slog" "github.com/gofiber/fiber/v2" + "github.com/zepyrshut/rating-orama/internal/app" "github.com/zepyrshut/rating-orama/internal/handlers" "github.com/zepyrshut/rating-orama/internal/repository" ) -//go:embed database/migrations -var database embed.FS - const version = "0.2.0-beta.20241116-4" const appName = "rating-orama" @@ -22,21 +19,26 @@ func init() { gob.Register(map[string]string{}) } +//go:embed database/migrations +var database embed.FS + func main() { - app := app.New(version) - r := fiber.New(fiber.Config{ + app := app.NewExtendedApp(appName, version, ".env") + app.Migrate(database) + f := fiber.New(fiber.Config{ AppName: appName, }) - - dbPool := db.NewPGXPool(app.Database.DataSource) - defer dbPool.Close() - q := repository.NewPGXRepo(dbPool) - h := handlers.New(app, q) - router(h, r) + pgxPool := db.NewPGXPool(app.Database.DataSource) + defer pgxPool.Close() + + r := repository.NewPGXRepo(pgxPool, app) + h := handlers.New(r, app) + router(h, f) slog.Info("server started", "port", "8080", "version", version) - if err := r.Listen(":8080"); err != nil { + err := f.Listen(":8080") + if err != nil { slog.Error("cannot start server", "error", err) } } diff --git a/core/go.mod b/core/go.mod index ebdfffa..ebb8181 100644 --- a/core/go.mod +++ b/core/go.mod @@ -54,7 +54,7 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.19.0 // indirect gopher-toolbox v0.0.0-00010101000000-000000000000 ) diff --git a/core/go.sum b/core/go.sum index 68c59db..cfd7155 100644 --- a/core/go.sum +++ b/core/go.sum @@ -168,8 +168,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/core/internal/app/app.go b/core/internal/app/app.go new file mode 100644 index 0000000..03d31be --- /dev/null +++ b/core/internal/app/app.go @@ -0,0 +1,16 @@ +package app + +import ( + "gopher-toolbox/app" +) + +type ExtendedApp struct { + app.App +} + +func NewExtendedApp(appName, version, envDirectory string) *ExtendedApp { + app := app.New(appName, version, envDirectory) + return &ExtendedApp{ + App: *app, + } +} diff --git a/core/internal/handlers/handlers.go b/core/internal/handlers/handlers.go index fc9c3a8..a2f7297 100644 --- a/core/internal/handlers/handlers.go +++ b/core/internal/handlers/handlers.go @@ -1,30 +1,18 @@ package handlers import ( - "net/http" - - "gopher-toolbox/app" - - "github.com/gofiber/fiber/v2" + "github.com/zepyrshut/rating-orama/internal/app" "github.com/zepyrshut/rating-orama/internal/repository" ) type Handlers struct { - app *app.App + app *app.ExtendedApp queries repository.ExtendedQuerier } -func New(app *app.App, q repository.ExtendedQuerier) *Handlers { +func New(r repository.ExtendedQuerier, app *app.ExtendedApp) *Handlers { return &Handlers{ app: app, - queries: q, + queries: r, } } - -func (hq *Handlers) ToBeImplemented(c *fiber.Ctx) error { - return c.Status(http.StatusNotImplemented).JSON("not implemented") -} - -func (hq *Handlers) Ping(c *fiber.Ctx) error { - return c.JSON("pong") -} diff --git a/core/internal/handlers/tvshow.go b/core/internal/handlers/tvshow.go index 9ca7123..22f74f0 100644 --- a/core/internal/handlers/tvshow.go +++ b/core/internal/handlers/tvshow.go @@ -13,6 +13,10 @@ import ( func (hq *Handlers) GetTVShow(c *fiber.Ctx) error { ttShowID := c.Query("ttid") + if ttShowID == "" { + return c.SendStatus(http.StatusBadRequest) + } + var title string var scraperEpisodes []scraper.Episode var sqlcEpisodes []sqlc.Episode @@ -20,7 +24,7 @@ func (hq *Handlers) GetTVShow(c *fiber.Ctx) error { tvShow, err := hq.queries.CheckTVShowExists(c.Context(), ttShowID) if err != nil { title, scraperEpisodes = scraper.ScrapeEpisodes(ttShowID) - // TODO: make transactional + //TODO: make transactional ttShow, err := hq.queries.CreateTVShow(c.Context(), sqlc.CreateTVShowParams{ TtImdb: ttShowID, Name: title, diff --git a/core/internal/repository/pgxrepo.go b/core/internal/repository/pgxrepo.go index c69b286..e10a3fe 100644 --- a/core/internal/repository/pgxrepo.go +++ b/core/internal/repository/pgxrepo.go @@ -2,39 +2,44 @@ package repository import ( "context" - "log/slog" + "fmt" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + "github.com/zepyrshut/rating-orama/internal/app" "github.com/zepyrshut/rating-orama/internal/sqlc" ) type pgxRepository struct { *sqlc.Queries - db *pgxpool.Pool + pool *pgxpool.Pool + app *app.ExtendedApp } -func NewPGXRepo(db *pgxpool.Pool) ExtendedQuerier { +var _ ExtendedQuerier = &pgxRepository{} + +func NewPGXRepo(pgx *pgxpool.Pool, app *app.ExtendedApp) ExtendedQuerier { return &pgxRepository{ - Queries: sqlc.New(db), - db: db, + Queries: sqlc.New(pgx), + pool: pgx, + app: app, } } -func (r *pgxRepository) execTx(ctx context.Context, txFunc func(tx pgx.Tx) error) error { - slog.Info("starting transaction", "txFunc", txFunc) - tx, err := r.db.Begin(ctx) +func (r *pgxRepository) execTx(ctx context.Context, fn func(*sqlc.Queries) error) error { + tx, err := r.pool.Begin(ctx) if err != nil { - slog.Error("failed to start transaction", "error", err) - return err - } - defer tx.Rollback(ctx) - - if err := txFunc(tx); err != nil { - slog.Error("failed to execute transaction", "error", err) return err } - slog.Info("committing transaction", "txFunc", txFunc) + q := sqlc.New(tx) + + err = fn(q) + if err != nil { + if rbErr := tx.Rollback(ctx); rbErr != nil { + return fmt.Errorf("tx err: %v, rb err: %v", err, rbErr) + } + return err + } + return tx.Commit(ctx) } diff --git a/core/internal/repository/querier.go b/core/internal/repository/querier.go index 4aa8859..7414b31 100644 --- a/core/internal/repository/querier.go +++ b/core/internal/repository/querier.go @@ -2,12 +2,12 @@ package repository import ( "context" - "github.com/zepyrshut/rating-orama/internal/scraper" + "github.com/zepyrshut/rating-orama/internal/scraper" "github.com/zepyrshut/rating-orama/internal/sqlc" ) type ExtendedQuerier interface { sqlc.Querier - CreateTvShowWithEpisodes(ctx context.Context, tvShow sqlc.CreateTVShowParams, episodes []scraper.Episode) ([]sqlc.Episode, error) + CreateTvShowWithEpisodesTX(ctx context.Context, tvShow sqlc.CreateTVShowParams, episodes []scraper.Episode) ([]sqlc.Episode, error) } diff --git a/core/internal/repository/tvshow.go b/core/internal/repository/tvshow.go deleted file mode 100644 index 3877528..0000000 --- a/core/internal/repository/tvshow.go +++ /dev/null @@ -1,33 +0,0 @@ -package repository - -import ( - "context" - "github.com/jackc/pgx/v5" - "github.com/zepyrshut/rating-orama/internal/scraper" - "github.com/zepyrshut/rating-orama/internal/sqlc" -) - -func (r *pgxRepository) CreateTvShowWithEpisodes(ctx context.Context, tvShow sqlc.CreateTVShowParams, episodes []scraper.Episode) ([]sqlc.Episode, error) { - var sqlcEpisodes []sqlc.Episode - err := r.execTx(ctx, func(tx pgx.Tx) error { - qtx := r.WithTx(tx) - tvShow, err := qtx.CreateTVShow(ctx, tvShow) - if err != nil { - return err - } - - for _, episode := range episodes { - sqlcEpisodeParams := episode.ToEpisodeParams(tvShow.ID) - episode, err := qtx.CreateEpisodes(ctx, sqlcEpisodeParams) - if err != nil { - return err - } - - sqlcEpisodes = append(sqlcEpisodes, episode) - } - - return nil - }) - - return sqlcEpisodes, err -} diff --git a/core/internal/repository/tvshow.tx.go b/core/internal/repository/tvshow.tx.go new file mode 100644 index 0000000..d9b4c30 --- /dev/null +++ b/core/internal/repository/tvshow.tx.go @@ -0,0 +1,34 @@ +package repository + +import ( + "context" + + "github.com/zepyrshut/rating-orama/internal/scraper" + "github.com/zepyrshut/rating-orama/internal/sqlc" +) + +func (r *pgxRepository) CreateTvShowWithEpisodesTX(ctx context.Context, tvShow sqlc.CreateTVShowParams, episodes []scraper.Episode) ([]sqlc.Episode, error) { + var err error + var episodesSqlc []sqlc.Episode + + err = r.execTx(ctx, func(tx *sqlc.Queries) error { + tvShow, err := tx.CreateTVShow(ctx, tvShow) + if err != nil { + return err + } + + for _, episode := range episodes { + sqlcEpisodeParams := episode.ToEpisodeParams(tvShow.ID) + episode, err := tx.CreateEpisodes(ctx, sqlcEpisodeParams) + if err != nil { + return err + } + + episodesSqlc = append(episodesSqlc, episode) + } + + return nil + }) + + return episodesSqlc, err +} diff --git a/core/internal/scraper/tvshow.go b/core/internal/scraper/tvshow.go index 133a426..9461d68 100644 --- a/core/internal/scraper/tvshow.go +++ b/core/internal/scraper/tvshow.go @@ -3,6 +3,7 @@ package scraper import ( "fmt" "log/slog" + "os" "regexp" "sort" "strconv" @@ -43,20 +44,6 @@ func (e Episode) ToEpisodeParams(tvShowID int32) sqlc.CreateEpisodesParams { } } -const ( - titleSelector = "h2.sc-b8cc654b-9.dmvgRY" - seasonsSelector = "ul.ipc-tabs a[data-testid='tab-season-entry']" - episodeCardSelector = "article.sc-f8507e90-1.cHtpvn.episode-item-wrapper" - seasonEpisodeAndTitleSelector = "div.ipc-title__text" - releasedDateSelector = "span.sc-f2169d65-10.bYaARM" - plotSelector = "div.ipc-html-content-inner-div" - starRatingSelector = "span.ipc-rating-star--rating" - voteCountSelector = "span.ipc-rating-star--voteCount" - imdbEpisodesURL = "https://www.imdb.com/title/%s/episodes/?season=%d" - visitURL = "https://www.imdb.com/title/%s/episodes" -) - - func ScrapeEpisodes(ttImdb string) (string, []Episode) { c := colly.NewCollector( colly.AllowedDomains("imdb.com", "www.imdb.com"), @@ -70,7 +57,7 @@ func ScrapeEpisodes(ttImdb string) (string, []Episode) { var seasons []int var title string - c.OnHTML(seasonsSelector, func(e *colly.HTMLElement) { + c.OnHTML(os.Getenv("SEASON_SELECTOR"), func(e *colly.HTMLElement) { seasonText := strings.TrimSpace(e.Text) seasonNum, err := strconv.Atoi(seasonText) if err == nil { @@ -78,7 +65,7 @@ func ScrapeEpisodes(ttImdb string) (string, []Episode) { } }) - c.OnHTML(titleSelector, func(e *colly.HTMLElement) { + c.OnHTML(os.Getenv("TITLE_SELECTOR"), func(e *colly.HTMLElement) { title = e.Text }) @@ -103,7 +90,7 @@ func ScrapeEpisodes(ttImdb string) (string, []Episode) { }) for _, seasonNum := range uniqueSeasons { - seasonURL := fmt.Sprintf(imdbEpisodesURL, ttImdb, seasonNum) + seasonURL := fmt.Sprintf(os.Getenv("IMDB_EPISODES_URL"), ttImdb, seasonNum) slog.Info("visiting season", "url", seasonURL) _ = episodeCollector.Visit(seasonURL) } @@ -111,7 +98,7 @@ func ScrapeEpisodes(ttImdb string) (string, []Episode) { episodeCollector.Wait() }) - _ = c.Visit(fmt.Sprintf(visitURL, ttImdb)) + _ = c.Visit(fmt.Sprintf(os.Getenv("VISIT_URL"), ttImdb)) c.Wait() slog.Info("scraped all seasons", "length", len(allSeasons)) @@ -126,26 +113,26 @@ func extractEpisodesFromSeason(data string) []Episode { } var episodes []Episode - doc.Find(episodeCardSelector).Each(func(i int, s *goquery.Selection) { + doc.Find(os.Getenv("EPISODE_CARD_SELECTOR")).Each(func(i int, s *goquery.Selection) { var episode Episode - seasonEpisodeTitle := s.Find(seasonEpisodeAndTitleSelector).Text() + seasonEpisodeTitle := s.Find(os.Getenv("SEASON_EPISODE_AND_TITLE_SELECTOR")).Text() episode.Season, episode.Episode, episode.Name = parseSeasonEpisodeTitle(seasonEpisodeTitle) - releasedDate := s.Find(releasedDateSelector).Text() + releasedDate := s.Find(os.Getenv("RELEASED_DATE_SELECTOR")).Text() episode.Released = parseReleasedDate(releasedDate) - plot := s.Find(plotSelector).Text() + plot := s.Find(os.Getenv("PLOT_SELECTOR")).Text() if plot == "Add a plot" { episode.Plot = "" } else { episode.Plot = plot } - starRating := s.Find(starRatingSelector).Text() + starRating := s.Find(os.Getenv("STAR_RATING_SELECTOR")).Text() episode.Rate = parseStarRating(starRating) - voteCount := s.Find(voteCountSelector).Text() + voteCount := s.Find(os.Getenv("VOTE_COUNT_SELECTOR")).Text() episode.VoteCount = parseVoteCount(voteCount) episodes = append(episodes, episode) diff --git a/core/internal/transfers/episodes.go b/core/internal/transfers/episodes.go new file mode 100644 index 0000000..a8b4927 --- /dev/null +++ b/core/internal/transfers/episodes.go @@ -0,0 +1,9 @@ +package transfers + +type EpisodePayload struct { + Title string + Season int + Episode int + Description string + Rating float64 +} diff --git a/core/docker/docker-compose.yml b/docker-compose.yml similarity index 100% rename from core/docker/docker-compose.yml rename to docker-compose.yml