Add photos_count field and optional extra-photos extraction

- Add PhotosCount field to Entry struct, extracted from darray[37][1]
- Add Date field to Image struct for photo publication dates
- Add -extra-photos CLI flag to enable enhanced photo extraction
- When enabled, extracts dates for category images and individual photos
- Wire ExtractExtraPhotos through job chain (GmapJob → PlaceJob)
- Update all runners (file, database, web, lambda) to support new flag

Photo dates are extracted from [21][6][8] as [year, month, day] and
formatted as YYYY-MM-DD.

Author: dredozubov

gmaps/entry.go

@@ -15,6 +15,7 @@ import (
 type Image struct {
 	Title string `json:"title"`
 	Image string `json:"image"`
+	Date  string `json:"date,omitempty"`
 }
 
 type LinkSource struct {
@@ -247,6 +248,93 @@ func (e *Entry) AddExtraReviews(pages [][]byte) {
 	}
 }
 
+// AddExtraPhotos extracts additional photo data including dates for existing images
+// and individual photos from the raw JSON data.
+func (e *Entry) AddExtraPhotos(raw []byte) {
+	var jd []any
+	if err := json.Unmarshal(raw, &jd); err != nil {
+		return
+	}
+
+	if len(jd) < 7 {
+		return
+	}
+
+	darray, ok := jd[6].([]any)
+	if !ok {
+		return
+	}
+
+	// Extract dates for existing category images from darray[171][0]
+	catImages := getNthElementAndCast[[]any](darray, 171, 0)
+	for i := range e.Images {
+		if i < len(catImages) {
+			cat := getNthElementAndCast[[]any](catImages, i)
+			// Date is at cat[3][0][21][6][8]
+			dateArr := getNthElementAndCast[[]any](cat, 3, 0, 21, 6, 8)
+			e.Images[i].Date = formatPhotoDate(dateArr)
+		}
+	}
+
+	// Extract individual photos from darray[37][0]
+	photoObjects := getNthElementAndCast[[]any](darray, 37, 0)
+	for i := range photoObjects {
+		photo := getNthElementAndCast[[]any](photoObjects, i)
+		if len(photo) == 0 {
+			continue
+		}
+
+		photoID := getNthElementAndCast[string](photo, 0)
+		photoURL := getNthElementAndCast[string](photo, 6, 0)
+		photoLabel := getNthElementAndCast[string](photo, 20)
+		dateArr := getNthElementAndCast[[]any](photo, 21, 6, 8)
+
+		if photoURL == "" {
+			continue
+		}
+
+		// Use label as title, fallback to "Photo"
+		title := photoLabel
+		if title == "" {
+			title = fmt.Sprintf("Photo %d", i+1)
+		}
+
+		// Check if this photo is already in Images (by URL or ID)
+		alreadyExists := false
+		for _, img := range e.Images {
+			if strings.Contains(img.Image, photoID) {
+				alreadyExists = true
+				break
+			}
+		}
+
+		if !alreadyExists {
+			e.Images = append(e.Images, Image{
+				Title: title,
+				Image: photoURL,
+				Date:  formatPhotoDate(dateArr),
+			})
+		}
+	}
+}
+
+// formatPhotoDate converts a date array [year, month, day, hour] to "YYYY-MM-DD" format.
+func formatPhotoDate(dateArr []any) string {
+	if len(dateArr) < 3 {
+		return ""
+	}
+
+	year := int(getNthElementAndCast[float64](dateArr, 0))
+	month := int(getNthElementAndCast[float64](dateArr, 1))
+	day := int(getNthElementAndCast[float64](dateArr, 2))
+
+	if year == 0 || month == 0 || day == 0 {
+		return ""
+	}
+
+	return fmt.Sprintf("%04d-%02d-%02d", year, month, day)
+}
+
 func extractReviews(data []byte) []Review {
 	// Skip the security prefix
 	prefix := ")]}'\n"

gmaps/job.go

@@ -29,6 +29,7 @@ type GmapJob struct {
 	Deduper             deduper.Deduper
 	ExitMonitor         exiter.Exiter
 	ExtractExtraReviews bool
+	ExtractExtraPhotos  bool
 }
 
 func NewGmapJob(
@@ -97,6 +98,12 @@ func WithExtraReviews() GmapJobOptions {
 	}
 }
 
+func WithExtraPhotos() GmapJobOptions {
+	return func(j *GmapJob) {
+		j.ExtractExtraPhotos = true
+	}
+}
+
 func (j *GmapJob) UseInResults() bool {
 	return false
 }
@@ -122,7 +129,7 @@ func (j *GmapJob) Process(ctx context.Context, resp *scrapemate.Response) (any,
 			jopts = append(jopts, WithPlaceJobExitMonitor(j.ExitMonitor))
 		}
 
-		placeJob := NewPlaceJob(j.ID, j.LangCode, resp.URL, j.ExtractEmail, j.ExtractExtraReviews, jopts...)
+		placeJob := NewPlaceJob(j.ID, j.LangCode, resp.URL, j.ExtractEmail, j.ExtractExtraReviews, j.ExtractExtraPhotos, jopts...)
 
 		next = append(next, placeJob)
 	} else {
@@ -133,7 +140,7 @@ func (j *GmapJob) Process(ctx context.Context, resp *scrapemate.Response) (any,
 					jopts = append(jopts, WithPlaceJobExitMonitor(j.ExitMonitor))
 				}
 
-				nextJob := NewPlaceJob(j.ID, j.LangCode, href, j.ExtractEmail, j.ExtractExtraReviews, jopts...)
+				nextJob := NewPlaceJob(j.ID, j.LangCode, href, j.ExtractEmail, j.ExtractExtraReviews, j.ExtractExtraPhotos, jopts...)
 
 				if j.Deduper == nil || j.Deduper.AddIfNotExists(ctx, href) {
 					next = append(next, nextJob)

gmaps/place.go

@@ -23,9 +23,10 @@ type PlaceJob struct {
 	ExtractEmail        bool
 	ExitMonitor         exiter.Exiter
 	ExtractExtraReviews bool
+	ExtractExtraPhotos  bool
 }
 
-func NewPlaceJob(parentID, langCode, u string, extractEmail, extraExtraReviews bool, opts ...PlaceJobOptions) *PlaceJob {
+func NewPlaceJob(parentID, langCode, u string, extractEmail, extraExtraReviews, extraPhotos bool, opts ...PlaceJobOptions) *PlaceJob {
 	const (
 		defaultPrio       = scrapemate.PriorityMedium
 		defaultMaxRetries = 3
@@ -46,6 +47,7 @@ func NewPlaceJob(parentID, langCode, u string, extractEmail, extraExtraReviews b
 	job.UsageInResultststs = true
 	job.ExtractEmail = extractEmail
 	job.ExtractExtraReviews = extraExtraReviews
+	job.ExtractExtraPhotos = extraPhotos
 
 	for _, opt := range opts {
 		opt(&job)
@@ -77,6 +79,10 @@ func (j *PlaceJob) Process(_ context.Context, resp *scrapemate.Response) (any, [
 		return nil, nil, err
 	}
 
+	if j.ExtractExtraPhotos {
+		entry.AddExtraPhotos(raw)
+	}
+
 	entry.ID = j.ParentID
 
 	if entry.Link == "" {

runner/databaserunner/databaserunner.go

@@ -152,6 +152,7 @@ func (d *dbrunner) produceSeedJobs(ctx context.Context) error {
 		nil,
 		nil,
 		d.cfg.ExtraReviews,
+		d.cfg.ExtraPhotos,
 	)
 	if err != nil {
 		return err

runner/filerunner/filerunner.go

@@ -88,6 +88,7 @@ func (r *fileRunner) Run(ctx context.Context) (err error) {
 		dedup,
 		exitMonitor,
 		r.cfg.ExtraReviews,
+		r.cfg.ExtraPhotos,
 	)
 	if err != nil {
 		return err

runner/jobs.go

@@ -28,6 +28,7 @@ func CreateSeedJobs(
 	dedup deduper.Deduper,
 	exitMonitor exiter.Exiter,
 	extraReviews bool,
+	extraPhotos bool,
 ) (jobs []scrapemate.IJob, err error) {
 	var lat, lon float64
 
@@ -100,6 +101,10 @@ func CreateSeedJobs(
 				opts = append(opts, gmaps.WithExtraReviews())
 			}
 
+			if extraPhotos {
+				opts = append(opts, gmaps.WithExtraPhotos())
+			}
+
 			job = gmaps.NewGmapJob(id, langCode, query, maxDepth, email, geoCoordinates, zoom, opts...)
 		} else {
 			jparams := gmaps.MapSearchParams{

runner/lambdaaws/invoker.go

@@ -129,6 +129,7 @@ func (i *invoker) setPayloads(cfg *runner.Config) error {
 				Language:     cfg.LangCode,
 				FunctionName: cfg.FunctionName,
 				ExtraReviews: cfg.ExtraReviews,
+				ExtraPhotos:  cfg.ExtraPhotos,
 			}
 			i.payloads = append(i.payloads, payload)
 
@@ -148,6 +149,7 @@ func (i *invoker) setPayloads(cfg *runner.Config) error {
 			Language:     cfg.LangCode,
 			FunctionName: cfg.FunctionName,
 			ExtraReviews: cfg.ExtraReviews,
+			ExtraPhotos:  cfg.ExtraPhotos,
 		}
 		i.payloads = append(i.payloads, payload)
 	}

runner/lambdaaws/io.go

@@ -11,4 +11,5 @@ type lInput struct {
 	FunctionName     string   `json:"function_name"`
 	DisablePageReuse bool     `json:"disable_page_reuse"`
 	ExtraReviews     bool     `json:"extra_reviews"`
+	ExtraPhotos      bool     `json:"extra_photos"`
 }

runner/lambdaaws/lambdaaws.go

@@ -90,6 +90,7 @@ func (l *lambdaAwsRunner) handler(ctx context.Context, input lInput) error {
 		nil,
 		exitMonitor,
 		input.ExtraReviews,
+		input.ExtraPhotos,
 	)
 	if err != nil {
 		return err

runner/runner.go

@@ -80,6 +80,7 @@ type Config struct {
 	DisablePageReuse         bool
 	ExtraReviews             bool
 	LeadsDBAPIKey            string
+	ExtraPhotos              bool
 }
 
 func ParseConfig() *Config {
@@ -127,6 +128,7 @@ func ParseConfig() *Config {
 	flag.BoolVar(&cfg.DisablePageReuse, "disable-page-reuse", false, "disable page reuse in playwright")
 	flag.BoolVar(&cfg.ExtraReviews, "extra-reviews", false, "enable extra reviews collection")
 	flag.StringVar(&cfg.LeadsDBAPIKey, "leadsdb-api-key", "", "LeadsDB API key for exporting results to LeadsDB")
+	flag.BoolVar(&cfg.ExtraPhotos, "extra-photos", false, "enable extra photos collection (includes dates and individual photos)")
 
 	flag.Parse()