From a4591aa177700b324851d3cb28497779f0f06645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20P=C3=A9rez?= Date: Wed, 6 Nov 2024 16:19:27 +0100 Subject: [PATCH] feat: :sparkles: almost done frontend missing yet --- core/cmd/routes.go | 13 ----- .../migrations/001_rating_schema.up.sql | 6 +-- core/database/queries/tv_show.sql | 2 +- core/internal/handlers/tvshow.go | 52 +++++++++++++++++-- core/internal/scraper/tvshow.go | 37 +++++++++---- core/internal/sqlc/models.go | 18 +++---- core/internal/sqlc/querier.go | 2 +- core/internal/sqlc/tv_show.sql.go | 22 ++++---- 8 files changed, 99 insertions(+), 53 deletions(-) diff --git a/core/cmd/routes.go b/core/cmd/routes.go index 32375a6..c9f917c 100644 --- a/core/cmd/routes.go +++ b/core/cmd/routes.go @@ -13,18 +13,5 @@ func Router(h *handlers.Handlers, app *config.App) *gin.Engine { r.GET("/tvshow", h.GetTVShow) - // app.Use(recover.New()) - - // app.Static("/js", "./views/js") - // app.Static("/css", "./views/css") - - // app.Get("/", handlers.Repo.Index) - // app.Get("/another", handlers.Repo.Another) - // app.Get("/tv-show", handlers.Repo.GetAllChapters) - - // dev := app.Group("/dev") - // dev.Get("/ping", handlers.Repo.Ping) - // dev.Get("/panic", handlers.Repo.Panic) - return r } diff --git a/core/database/migrations/001_rating_schema.up.sql b/core/database/migrations/001_rating_schema.up.sql index 4a8abc7..3e40280 100644 --- a/core/database/migrations/001_rating_schema.up.sql +++ b/core/database/migrations/001_rating_schema.up.sql @@ -1,5 +1,5 @@ create table if not exists tv_show ( - id integer primary key, + id serial unique primary key, "name" varchar not null, tt_imdb varchar not null, popularity int not null default 0, @@ -13,7 +13,7 @@ create index if not exists idx_tv_show_tt_imdb on "tv_show" ("tt_imdb"); create index if not exists idx_tv_show_updated_at on "tv_show" ("updated_at"); create table if not exists episodes ( - id integer primary key, + id serial unique primary key, tv_show_id integer not null, season integer not null, @@ -21,7 +21,7 @@ create table if not exists episodes ( released date, "name" varchar not null, plot text not null default '', - avg_rating numeric(1,1) not null default 0, + avg_rating real not null default 0, vote_count int not null default 0, foreign key (tv_show_id) references tv_show (id) diff --git a/core/database/queries/tv_show.sql b/core/database/queries/tv_show.sql index dd063bb..6090617 100644 --- a/core/database/queries/tv_show.sql +++ b/core/database/queries/tv_show.sql @@ -18,7 +18,7 @@ where tv_show_id = $1; -- name: IncreasePopularity :exec update "tv_show" set popularity = popularity + 1 -where id = $1; +where tt_imdb = $1; -- name: TvShowAverageRating :one select avg(avg_rating) from "episodes" diff --git a/core/internal/handlers/tvshow.go b/core/internal/handlers/tvshow.go index ab560ab..21862b9 100644 --- a/core/internal/handlers/tvshow.go +++ b/core/internal/handlers/tvshow.go @@ -6,18 +6,60 @@ import ( "github.com/gin-gonic/gin" "github.com/zepyrshut/rating-orama/internal/scraper" + "github.com/zepyrshut/rating-orama/internal/sqlc" ) func (hq *Handlers) GetTVShow(c *gin.Context) { ttShowID := c.Query("ttid") - slog.Info("GetTVShow", "ttid", ttShowID) + slog.Info("", "ttid", ttShowID, RequestID, c.Request.Context().Value(RequestID)) - title, seasons := scraper.ScrapeSeasons(ttShowID) + var title string + var scraperEpisodes []scraper.Episode + var sqlcEpisodes []sqlc.Episode - slog.Info("scraped seasons", "ttid", ttShowID, "title", title) + tvShow, err := hq.Queries.CheckTVShowExists(c, ttShowID) + if err != nil { + title, scraperEpisodes = scraper.ScrapeEpisodes(ttShowID) + // TODO: make transactional + ttShow, err := hq.Queries.CreateTVShow(c, sqlc.CreateTVShowParams{ + TtImdb: ttShowID, + Name: title, + }) + if err != nil { + slog.Error("failed to create tv show", "ttid", ttShowID, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": ErrorCreating}) + } + + slog.Info("ttshowid", "id", ttShow.ID) + for _, episode := range scraperEpisodes { + sqlcEpisodesParams := episode.ToEpisodeParams(ttShow.ID) + sqlcEpisode, err := hq.Queries.CreateEpisodes(c, sqlcEpisodesParams) + if err != nil { + slog.Error("failed to create episodes", "ttid", ttShowID, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": ErrorCreating}) + return + } + + sqlcEpisodes = append(sqlcEpisodes, sqlcEpisode) + } + + slog.Info("scraped seasons", "ttid", ttShowID, "title", title) + } else { + title = tvShow.Name + sqlcEpisodes, err = hq.Queries.GetEpisodes(c, tvShow.ID) + if err != nil { + slog.Error("failed to get episodes", "ttid", ttShowID, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": ErrorGetting}) + return + } + + hq.Queries.IncreasePopularity(c, ttShowID) + slog.Info("tv show exists", "ttid", ttShowID, "title", tvShow.Name) + } c.JSON(http.StatusOK, gin.H{ - "title": title, - "seasons": seasons, + "popularity": tvShow.Popularity, + "title": title, + "seasons": sqlcEpisodes, }) } diff --git a/core/internal/scraper/tvshow.go b/core/internal/scraper/tvshow.go index 7832914..6e4fb74 100644 --- a/core/internal/scraper/tvshow.go +++ b/core/internal/scraper/tvshow.go @@ -10,6 +10,8 @@ import ( "time" "github.com/gocolly/colly" + "github.com/jackc/pgx/v5/pgtype" + "github.com/zepyrshut/rating-orama/internal/sqlc" ) type Episode struct { @@ -18,11 +20,26 @@ type Episode struct { Released time.Time Name string Plot string - Rate float64 + Rate float32 VoteCount int } -type Season []Episode +func (e Episode) ToEpisodeParams(tvShowID int32) sqlc.CreateEpisodesParams { + + var date pgtype.Date + date.Scan(e.Released) + + return sqlc.CreateEpisodesParams{ + TvShowID: tvShowID, + Season: int32(e.Season), + Episode: int32(e.Episode), + Name: e.Name, + Released: date, + Plot: e.Plot, + AvgRating: e.Rate, + VoteCount: int32(e.VoteCount), + } +} const ( titleSelector = "h2.sc-b8cc654b-9.dmvgRY" @@ -33,7 +50,7 @@ const ( visitURL = "https://www.imdb.com/title/%s/episodes" ) -func ScrapeSeasons(ttImdb string) (string, []Season) { +func ScrapeEpisodes(ttImdb string) (string, []Episode) { c := colly.NewCollector( colly.AllowedDomains("imdb.com", "www.imdb.com"), ) @@ -42,7 +59,7 @@ func ScrapeSeasons(ttImdb string) (string, []Season) { r.Headers.Set("Accept-Language", "en-US") }) - var allSeasons []Season + var allSeasons []Episode var seasons []int var title string @@ -75,8 +92,7 @@ func ScrapeSeasons(ttImdb string) (string, []Season) { episodeCollector.OnHTML(episodesSelector, func(e *colly.HTMLElement) { seasonEpisodes := extractEpisodesFromSeason(e.Text) - allSeasons = append(allSeasons, seasonEpisodes) - //allEpisodes = append(allEpisodes, seasonEpisodes...) + allSeasons = append(allSeasons, seasonEpisodes...) }) for _, seasonNum := range uniqueSeasons { @@ -94,7 +110,7 @@ func ScrapeSeasons(ttImdb string) (string, []Season) { return title, allSeasons } -func extractEpisodesFromSeason(data string) Season { +func extractEpisodesFromSeason(data string) []Episode { const pattern = `(S\d+\.E\d+)\s∙\s(.*?)` + `(Mon|Tue|Wed|Thu|Fri|Sat|Sun),\s` + `(.*?),\s(\d{4})(.*?)` + @@ -123,15 +139,16 @@ func extractEpisodesFromSeason(data string) Season { seasonNum := strings.TrimPrefix(strings.Split(seasonEpisode, ".")[0], "S") episodeNum := strings.TrimPrefix(strings.Split(seasonEpisode, ".")[1], "E") - votes, _ := strconv.Atoi(strings.TrimSuffix(strings.TrimSuffix(voteCount, "K"), "K")) + votesInt, _ := strconv.Atoi(strings.TrimSuffix(strings.TrimSuffix(voteCount, "K"), "K")) + rateFloat, _ := strconv.ParseFloat(strings.TrimSuffix(rate, "/10"), 32) episode.Name = name episode.Episode, _ = strconv.Atoi(episodeNum) episode.Season, _ = strconv.Atoi(seasonNum) episode.Released, _ = time.Parse("Mon, Jan 2, 2006", fmt.Sprintf("%s, %s, %s", day, dateRest, year)) episode.Plot = plot - episode.Rate, _ = strconv.ParseFloat(strings.TrimSuffix(rate, "/10"), 2) - episode.VoteCount = votes * 1000 + episode.Rate = float32(rateFloat) + episode.VoteCount = votesInt * 1000 episodes = append(episodes, episode) } diff --git a/core/internal/sqlc/models.go b/core/internal/sqlc/models.go index a4b6e39..ba37523 100644 --- a/core/internal/sqlc/models.go +++ b/core/internal/sqlc/models.go @@ -9,15 +9,15 @@ import ( ) type Episode struct { - ID int32 `json:"id"` - TvShowID int32 `json:"tv_show_id"` - Season int32 `json:"season"` - Episode int32 `json:"episode"` - Released pgtype.Date `json:"released"` - Name string `json:"name"` - Plot string `json:"plot"` - AvgRating pgtype.Numeric `json:"avg_rating"` - VoteCount int32 `json:"vote_count"` + ID int32 `json:"id"` + TvShowID int32 `json:"tv_show_id"` + Season int32 `json:"season"` + Episode int32 `json:"episode"` + Released pgtype.Date `json:"released"` + Name string `json:"name"` + Plot string `json:"plot"` + AvgRating float32 `json:"avg_rating"` + VoteCount int32 `json:"vote_count"` } type TvShow struct { diff --git a/core/internal/sqlc/querier.go b/core/internal/sqlc/querier.go index 80aed3d..49febba 100644 --- a/core/internal/sqlc/querier.go +++ b/core/internal/sqlc/querier.go @@ -13,7 +13,7 @@ type Querier interface { CreateEpisodes(ctx context.Context, arg CreateEpisodesParams) (Episode, error) CreateTVShow(ctx context.Context, arg CreateTVShowParams) (TvShow, error) GetEpisodes(ctx context.Context, tvShowID int32) ([]Episode, error) - IncreasePopularity(ctx context.Context, id int32) error + IncreasePopularity(ctx context.Context, ttImdb string) error SeasonAverageRating(ctx context.Context, arg SeasonAverageRatingParams) (float64, error) TvShowAverageRating(ctx context.Context, tvShowID int32) (float64, error) TvShowMedianRating(ctx context.Context, tvShowID int32) (float64, error) diff --git a/core/internal/sqlc/tv_show.sql.go b/core/internal/sqlc/tv_show.sql.go index 8c9ede4..a11fc89 100644 --- a/core/internal/sqlc/tv_show.sql.go +++ b/core/internal/sqlc/tv_show.sql.go @@ -37,14 +37,14 @@ returning id, tv_show_id, season, episode, released, name, plot, avg_rating, vot ` type CreateEpisodesParams struct { - TvShowID int32 `json:"tv_show_id"` - Season int32 `json:"season"` - Episode int32 `json:"episode"` - Released pgtype.Date `json:"released"` - Name string `json:"name"` - Plot string `json:"plot"` - AvgRating pgtype.Numeric `json:"avg_rating"` - VoteCount int32 `json:"vote_count"` + TvShowID int32 `json:"tv_show_id"` + Season int32 `json:"season"` + Episode int32 `json:"episode"` + Released pgtype.Date `json:"released"` + Name string `json:"name"` + Plot string `json:"plot"` + AvgRating float32 `json:"avg_rating"` + VoteCount int32 `json:"vote_count"` } func (q *Queries) CreateEpisodes(ctx context.Context, arg CreateEpisodesParams) (Episode, error) { @@ -135,11 +135,11 @@ func (q *Queries) GetEpisodes(ctx context.Context, tvShowID int32) ([]Episode, e const increasePopularity = `-- name: IncreasePopularity :exec update "tv_show" set popularity = popularity + 1 -where id = $1 +where tt_imdb = $1 ` -func (q *Queries) IncreasePopularity(ctx context.Context, id int32) error { - _, err := q.db.Exec(ctx, increasePopularity, id) +func (q *Queries) IncreasePopularity(ctx context.Context, ttImdb string) error { + _, err := q.db.Exec(ctx, increasePopularity, ttImdb) return err }