feat(ds-host): add FetchDatetime to listing metadata

cmd/ds-host/appops/remote.go

@@ -16,11 +16,6 @@ import (
 	"github.com/teleclimber/DropServer/cmd/ds-host/record"
 )
 
-type cachedListing struct {
-	listingFetch domain.AppListingFetch
-	fetchDt      time.Time
-}
-
 const cacheDuration = time.Minute * 10
 
 type RemoteAppGetter struct {
@@ -44,7 +39,7 @@ type RemoteAppGetter struct {
 	ticker *time.Ticker
 	stop   chan struct{}
 
-	listingCache map[string]cachedListing
+	listingCache map[string]domain.AppListingFetch
 }
 
 const refreshTickerInterval = time.Hour
@@ -90,7 +85,7 @@ func (r *RemoteAppGetter) init() {
 		CheckRedirect: checkRedirect,
 		Transport:     transport}
 
-	r.listingCache = make(map[string]cachedListing)
+	r.listingCache = make(map[string]domain.AppListingFetch)
 }
 
 func checkRedirect(req *http.Request, via []*http.Request) error {
@@ -162,25 +157,25 @@ func (r *RemoteAppGetter) RefreshAppListing(appID domain.AppID) error {
 		return err
 	}
 	if listingFetch.NotModified {
-		r.AppModel.SetLastFetch(appID, time.Now(), "not-modified")
+		r.AppModel.SetLastFetch(appID, listingFetch.FetchDatetime, "not-modified")
 		return nil
 	}
 	if listingFetch.NewURL != "" {
-		r.AppModel.SetNewUrl(appID, listingFetch.NewURL, time.Now())
+		r.AppModel.SetNewUrl(appID, listingFetch.NewURL, listingFetch.FetchDatetime)
 		return errors.New("app listing has moved, new URL: " + listingFetch.NewURL)
 	}
 
 	err = ValidateListing(listingFetch.Listing)
 	if err != nil {
-		r.AppModel.SetLastFetch(appID, time.Now(), "error")
+		r.AppModel.SetLastFetch(appID, listingFetch.FetchDatetime, "error")
 		err = fmt.Errorf("error validating listing from %v: %w", appUrlData.URL, err)
 		return err
 	}
 
 	latestVersion, err := GetLatestVersion(listingFetch.Listing.Versions)
 	if err != nil {
 		// this is a coding error: we validated the listing earlier so there is no reason to get an error here.
-		r.AppModel.SetLastFetch(appID, time.Now(), "error")
+		r.AppModel.SetLastFetch(appID, listingFetch.FetchDatetime, "error")
 		r.getLogger("RefreshAppListing").AddNote("Unexpected error fron GetLatestVersion").Error(err)
 		return fmt.Errorf("unexpected error: %w", err)
 	}
@@ -238,9 +233,7 @@ func (r *RemoteAppGetter) FetchValidListing(url string) (domain.AppListingFetch,
 	}
 	listingFetch.LatestVersion = latestVersion
 
-	r.listingCache[url] = cachedListing{
-		listingFetch: listingFetch,
-		fetchDt:      time.Now()}
+	r.listingCache[url] = listingFetch
 
 	return listingFetch, nil
 }
@@ -249,8 +242,8 @@ func (r *RemoteAppGetter) FetchValidListing(url string) (domain.AppListingFetch,
 func (r *RemoteAppGetter) FetchCachedListing(url string) (domain.AppListingFetch, error) {
 	r.init()
 	cached, ok := r.listingCache[url]
-	if ok && isFresh(cached.fetchDt) {
-		return cached.listingFetch, nil
+	if ok && isFresh(cached.FetchDatetime) {
+		return cached, nil
 	}
 	listingFetch, err := r.FetchValidListing(url)
 	return listingFetch, err
@@ -288,6 +281,8 @@ func (r *RemoteAppGetter) fetchListing(url string, etag string) (domain.AppListi
 		req.Header.Set("If-None-Match", etag)
 	}
 
+	fetchDt := time.Now()
+
 	resp, err := r.client.Do(req)
 	if err != nil {
 		// Error is of type url.Error: timeouts, etc...
@@ -306,18 +301,18 @@ func (r *RemoteAppGetter) fetchListing(url string, etag string) (domain.AppListi
 		if newURL == "" {
 			return domain.AppListingFetch{}, fmt.Errorf("got response code %v but the Location header is empty", resp.StatusCode)
 		}
-		return domain.AppListingFetch{NewURL: newURL}, nil
+		return domain.AppListingFetch{FetchDatetime: fetchDt, NewURL: newURL}, nil
 	}
 	if resp.StatusCode == http.StatusNotModified {
-		return domain.AppListingFetch{NotModified: true}, nil
+		return domain.AppListingFetch{FetchDatetime: fetchDt, NotModified: true}, nil
 	}
 	if resp.StatusCode != http.StatusOK {
 		return domain.AppListingFetch{}, fmt.Errorf("got response code: %v", resp.Status)
 	}
 
 	newEtag := resp.Header.Get("ETag")
 	lm := resp.Header.Get("Last-Modified")
-	newLastModified := time.Now()
+	newLastModified := fetchDt
 	if lm != "" {
 		lmt, err := http.ParseTime(lm)
 		if err != nil {
@@ -338,6 +333,7 @@ func (r *RemoteAppGetter) fetchListing(url string, etag string) (domain.AppListi
 	}
 
 	return domain.AppListingFetch{
+		FetchDatetime:   fetchDt,
 		Listing:         listing,
 		Etag:            newEtag,
 		ListingDatetime: newLastModified,

cmd/ds-host/domain/domain.go

@@ -370,6 +370,8 @@ type App struct {
 
 // AppListingFetch is the app listing along with some fetch metadata
 type AppListingFetch struct {
+	// FetchDatetime is the time that the fetch was performed
+	FetchDatetime time.Time
 	// NotModified is true if the remote endpoint returned the Not-Modified header
 	NotModified bool
 	// NewURL is set if remote endpoint returns a redirect

cmd/ds-host/models/appmodel/appmodel.go

@@ -215,7 +215,7 @@ func (m *AppModel) CreateFromURL(ownerID domain.UserID, url string, auto bool, l
 		return domain.AppID(0), err
 	}
 
-	err = setLast(appID, "ok", listingFetch.ListingDatetime, tx)
+	err = setLast(appID, "ok", listingFetch.FetchDatetime, tx)
 	if err != nil {
 		m.getLogger("CreateFromURL(), setLast()").Error(err)
 		return domain.AppID(0), err
@@ -452,7 +452,7 @@ func (m *AppModel) SetListing(appID domain.AppID, listingFetch domain.AppListing
 	defer tx.Rollback()
 
 	// set last fetch data
-	err = setLast(appID, "ok", listingFetch.ListingDatetime, tx)
+	err = setLast(appID, "ok", listingFetch.FetchDatetime, tx)
 	if err != nil {
 		m.getLogger("SetListing(), setLast()").AppID(appID).Error(err)
 		return err
@@ -504,7 +504,7 @@ func (m *AppModel) SetNewUrl(appID domain.AppID, url string, dt time.Time) error
 }
 
 // UpdateURL of app listing and set the listing.
-func (m *AppModel) UpdateURL(appID domain.AppID, url string, listingFetch domain.AppListingFetch) error { // this should include the listing found at theat url and any relevant meta, like when updating listing
+func (m *AppModel) UpdateURL(appID domain.AppID, url string, listingFetch domain.AppListingFetch) error {
 	tx, err := m.DB.Handle.Beginx()
 	if err != nil {
 		m.getLogger("SetListing(), Beginx()").Error(err)
@@ -524,7 +524,7 @@ func (m *AppModel) UpdateURL(appID domain.AppID, url string, listingFetch domain
 	}
 
 	// set last fetch data
-	err = setLast(appID, "ok", listingFetch.ListingDatetime, tx)
+	err = setLast(appID, "ok", listingFetch.FetchDatetime, tx)
 	if err != nil {
 		m.getLogger("UpdateURL(), setLast()").AppID(appID).Error(err)
 		return err

cmd/ds-host/models/appmodel/appmodel_test.go

@@ -358,6 +358,7 @@ func TestCreateAppUrlData(t *testing.T) {
 	}
 
 	lf := domain.AppListingFetch{
+		FetchDatetime:   dt,
 		Listing:         expectedListing,
 		ListingDatetime: dt,
 		Etag:            expected.Etag,
@@ -447,6 +448,7 @@ func TestAppUrlUpdates(t *testing.T) {
 	url := "abc.com/app"
 
 	lf := domain.AppListingFetch{
+		FetchDatetime:   dt1,
 		Listing:         domain.AppListing{},
 		ListingDatetime: dt1,
 		Etag:            "etag1",
@@ -526,6 +528,7 @@ func TestSetListing(t *testing.T) {
 	url := "abc.com/app"
 
 	lf := domain.AppListingFetch{
+		FetchDatetime:   dt1,
 		Listing:         domain.AppListing{},
 		ListingDatetime: dt1,
 		Etag:            "etag1",
@@ -559,6 +562,7 @@ func TestSetListing(t *testing.T) {
 				Icon:      "icon-1.5.0.gif"},
 		},
 	}
+	lf.FetchDatetime = dt2
 	lf.Listing = expectedListing
 	lf.ListingDatetime = dt2
 	lf.Etag = "etag2"
@@ -604,6 +608,7 @@ func TestUpdateUrl(t *testing.T) {
 	url2 := "new.site/app"
 
 	lf := domain.AppListingFetch{
+		FetchDatetime:   dt1,
 		Listing:         domain.AppListing{},
 		ListingDatetime: dt1,
 		Etag:            "etag1",
@@ -636,6 +641,7 @@ func TestUpdateUrl(t *testing.T) {
 				Icon:      "icon-1.5.0.gif"},
 		},
 	}
+	lf.FetchDatetime = dt2
 	lf.Listing = expectedListing
 	lf.ListingDatetime = dt2
 	lf.Etag = "etag2"