diff --git a/dupcmd/duplicate.go b/dupcmd/duplicate.go new file mode 100644 index 00000000..a05d00dc --- /dev/null +++ b/dupcmd/duplicate.go @@ -0,0 +1,156 @@ +/* +Check the list of photos to list and discard duplicates. +*/ +package dupcmd + +import ( + "context" + "flag" + "immich-go/immich" + "immich-go/immich/logger" + "sort" + "strings" + "time" +) + +type DuplicateCmd struct { + logger *logger.Logger + Immich *immich.ImmichClient // Immich client + + DryRun bool // Display actions but don't change anything + DateRange immich.DateRange // Set capture date range +} + +type duplicateKey struct { + Date time.Time + Name string +} + +func NewDuplicateCmd(ctx context.Context, ic *immich.ImmichClient, logger *logger.Logger, args []string) (*DuplicateCmd, error) { + cmd := flag.NewFlagSet("upload", flag.ExitOnError) + validRange := immich.DateRange{} + validRange.Set("1850-01-04,3000-01-01") + app := DuplicateCmd{ + logger: logger, + Immich: ic, + DateRange: validRange, + } + + cmd.BoolVar(&app.DryRun, "dry-run", true, "display actions but don't touch source or destination") + cmd.Var(&app.DateRange, "date", "Process only document having a capture date in that range.") + err := cmd.Parse(args) + return &app, err +} + +func DuplicateCommand(ctx context.Context, ic *immich.ImmichClient, log *logger.Logger, args []string) error { + app, err := NewDuplicateCmd(ctx, ic, log, args) + if err != nil { + return err + } + + log.MessageContinue(logger.OK, "Get server's assets...") + var list []*immich.Asset + list, err = app.Immich.GetAllAssets(ctx, nil) + if err != nil { + return err + } + log.MessageTerminate(logger.OK, "%d received", len(list)) + + log.MessageContinue(logger.OK, "Analyzing...") + duplicate := map[duplicateKey][]*immich.Asset{} + + count := 0 + dupCount := 0 + for _, a := range list { + select { + case <-ctx.Done(): + return ctx.Err() + default: + count++ + if app.DateRange.InRange(a.ExifInfo.DateTimeOriginal) { + k := duplicateKey{ + Date: a.ExifInfo.DateTimeOriginal, + Name: a.OriginalFileName, + } + l := duplicate[k] + if len(l) > 0 { + dupCount++ + } + l = append(l, a) + duplicate[k] = l + } + if count%253 == 0 { + log.Progress("%d medias, %d duplicate(s)...", count, dupCount) + } + } + } + log.Progress("%d medias, %d duplicate(s)...", count, dupCount) + log.MessageTerminate(logger.OK, " analyze completed.") + + keys := []duplicateKey{} + for k, l := range duplicate { + if len(l) < 2 { + continue + } + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + switch keys[i].Date.Compare(keys[j].Date) { + case -1: + return true + case +1: + return false + } + return strings.Compare(keys[i].Name, keys[j].Name) == -1 + }) + + for _, k := range keys { + select { + case <-ctx.Done(): + return ctx.Err() + default: + app.logger.Info("%s:", k.Date.Format(time.RFC3339)) + l := duplicate[k] + albums := []string{} + delete := []string{} + sort.Slice(l, func(i, j int) bool { return l[i].ExifInfo.FileSizeInByte < l[j].ExifInfo.FileSizeInByte }) + for p, a := range duplicate[k] { + if p < len(l)-1 { + log.Info(" %s(%s) %dx%d, %d bytes: delete", a.OriginalFileName, a.ID, a.ExifInfo.ExifImageWidth, a.ExifInfo.ExifImageHeight, a.ExifInfo.FileSizeInByte) + delete = append(delete, a.ID) + r, err := app.Immich.GetAssetAlbums(ctx, a.ID) + if err != nil { + log.Error("Can't get asset's albums: %s", err.Error()) + } else { + for _, al := range r { + albums = append(albums, al.ID) + } + } + } else { + log.Info(" %s(%s) %dx%d, %d bytes: keep", a.OriginalFileName, a.ID, a.ExifInfo.ExifImageWidth, a.ExifInfo.ExifImageHeight, a.ExifInfo.FileSizeInByte) + if !app.DryRun { + log.OK("Deleting following assets: %s", strings.Join(delete, ",")) + _, err = app.Immich.DeleteAssets(ctx, delete) + if err != nil { + log.Error("Can't delete asset: %s", err.Error()) + } + } else { + log.Info("Skip deleting following %s, dry run mode", strings.Join(delete, ",")) + } + for _, al := range albums { + if !app.DryRun { + log.OK("Adding %s to album %s", a.ID, al) + _, err = app.Immich.AddAssetToAlbum(ctx, al, []string{a.ID}) + if err != nil { + log.Error("Can't delete asset: %s", err.Error()) + } + } else { + log.OK("Skip Adding %s to album %s, dry run mode", a.ID, al) + } + } + } + } + } + } + return nil +} diff --git a/immich/albums.go b/immich/albums.go index 4aa15327..79d1e375 100644 --- a/immich/albums.go +++ b/immich/albums.go @@ -125,3 +125,11 @@ func (ic *ImmichClient) CreateAlbum(ctx context.Context, name string, assets []s } return r, nil } + +func (ic *ImmichClient) GetAssetAlbums(ctx context.Context, id string) ([]AlbumSimplified, error) { + var r []AlbumSimplified + err := ic.newServerCall(ctx, "GetAssetAlbums").do( + get("/album?assetId="+id, setAcceptJSON()), + responseJSON(&r)) + return r, err +} diff --git a/immich/asset.go b/immich/asset.go index ae4dc812..887f62df 100644 --- a/immich/asset.go +++ b/immich/asset.go @@ -190,7 +190,7 @@ type deleteResponse []struct { Status string `json:"status"` } -func (ic *ImmichClient) DeleteAsset(ctx context.Context, id []string) (*deleteResponse, error) { +func (ic *ImmichClient) DeleteAssets(ctx context.Context, id []string) (*deleteResponse, error) { req := struct { IDs []string `json:"ids"` }{ diff --git a/main.go b/main.go index 71b9ece0..40a053bb 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "immich-go/dupcmd" "immich-go/immich" "immich-go/immich/logger" "immich-go/upcmd" @@ -110,6 +111,8 @@ func Run(ctx context.Context) error { switch cmd { case "upload": err = upcmd.UploadCommand(ctx, app.Immich, app.Logger, flag.Args()[1:]) + case "duplicate": + err = dupcmd.DuplicateCommand(ctx, app.Immich, app.Logger, flag.Args()[1:]) default: err = fmt.Errorf("unknwon command: %q", cmd) } diff --git a/readme.md b/readme.md index fe0f7a9b..22ab733c 100644 --- a/readme.md +++ b/readme.md @@ -58,6 +58,7 @@ immich-go -server URL -key KEY COMMAND -options... folder1|zip1 folder2|zip2.. ## Command `upload` +Use this command for uploading photos and videos from a local directory, a zipped folder or all zip files that google photo takeout procedure has generated. ### Switches and options: `-album "ALBUM NAME"` Import assets into the Immich album `ALBUM NAME`.
`-device-uuid VALUE` Force the device identification (default $HOSTNAME).
@@ -81,7 +82,7 @@ Specialized options for Google Photos management: `-keep-trashed ` Determines whether to import trashed images (default: FALSE).
-## Example Usage +### Example Usage: uploading a Google photos takeout archive To illustrate, here's a command importing photos from a Google Photos takeout archive captured between June 1st and June 30th, 2019, while auto-generating albums: @@ -90,6 +91,25 @@ To illustrate, here's a command importing photos from a Google Photos takeout ar -create-albums -google-photos -date=2019-06 ~/Download/takeout-20230715T073439Z-001.zip ~/Download/takeout-20230715T073439Z-002.zip ``` +## Command `duplicate` + +Use this command for analyzing the content of your `immich` server to find any files that share the same file name, the date of capture, but having different size. +Before deleting the inferior copies, the system get all albums they belong to, and add the superior copy to them. + +### Switches and options: +`-dry-run` Preview all actions as they would be done (default: TRUE).
+ + +### Example Usage: clean the `immich` server after having merged a google photo archive and original files + +This command examine the immich server content, remove less quality images, and preserve albums. + +NOTE: You should disable the dry run mode explicitly. + +```sh +./immich-go -server=http://mynas:2283 -key=zzV6k65KGLNB9mpGeri9n8Jk1VaNGHSCdoH1dY8jQ duplicate -dry-run=false +``` + # Merging strategy @@ -208,3 +228,9 @@ Additionally, deploying a Node.js program on user machines presents challenges. - [ ] samba - [ ] import remote folder +# Wanted features +- [ ] Import instead of Upload +- [x] Cleaning duplicates +- [ ] Set GPS location for images taken with a GPS-less camera based on + - [ ] Google location history + - [ ] KML,GPX track files diff --git a/upcmd/upload.go b/upcmd/upload.go index 3ac5803f..66db1c7a 100644 --- a/upcmd/upload.go +++ b/upcmd/upload.go @@ -184,7 +184,7 @@ func (app *UpCmd) handleAsset(ctx context.Context, a *assets.LocalAssetFile) err if app.DateRange.IsSet() { d, err := a.DateTaken() if err != nil { - app.logger.Error("Can't get capture date of the file. File %q skiped", a.FileName) + app.logger.Error("Can't get capture date of the file. File %q skipped", a.FileName) return nil } if !app.DateRange.InRange(d) { @@ -243,7 +243,7 @@ func (app *UpCmd) handleAsset(ctx context.Context, a *assets.LocalAssetFile) err func NewUpCmd(ctx context.Context, ic *immich.ImmichClient, logger *logger.Logger, args []string) (*UpCmd, error) { var err error - cmd := flag.NewFlagSet("generate", flag.ExitOnError) + cmd := flag.NewFlagSet("upload", flag.ExitOnError) app := UpCmd{ updateAlbums: map[string][]string{}, @@ -354,7 +354,7 @@ func (app *UpCmd) DeleteServerAssets(ctx context.Context, ids []string) error { app.logger.Warning("%d server assets to delete.", len(ids)) if !app.DryRun { - _, err := app.Immich.DeleteAsset(ctx, ids) + _, err := app.Immich.DeleteAssets(ctx, ids) return err } app.logger.Warning("%d server assets to delete. skipped dry-run mode", len(ids))