Merge pull request #414 from rusq/i268-avatars

i268 avatars

cmd/slackdump/internal/archive/archive.go

@@ -77,14 +77,26 @@ func RunArchive(ctx context.Context, cmd *base.Command, args []string) error {
 		lg,
 	)
 	defer stop()
-	// archive format has files stored in mattermost format.
-	subproc := fileproc.NewExport(fileproc.STmattermost, dl)
+	avdl, avstop := fileproc.NewDownloader(
+		ctx,
+		cfg.DownloadAvatars,
+		sess.Client(),
+		fsadapter.NewDirectory(cd.Name()),
+		lg,
+	)
+	defer avstop()
+	var (
+		// archive format has files stored in mattermost format.
+		subproc = fileproc.NewExport(fileproc.STmattermost, dl)
+		avproc  = fileproc.NewAvatarProc(avdl)
+	)
 	ctrl := control.New(
 		cd,
 		stream,
 		control.WithLogger(lg),
 		control.WithFiler(subproc),
 		control.WithFlags(control.Flags{MemberOnly: cfg.MemberOnly, RecordFiles: cfg.RecordFiles}),
+		control.WithAvatarProcessor(avproc),
 	)
 	if err := ctrl.Run(ctx, list); err != nil {
 		base.SetExitStatus(base.SApplicationError)

cmd/slackdump/internal/archive/archive_wizard.go

@@ -35,6 +35,7 @@ func configuration() cfgui.Configuration {
 				cfgui.ChannelIDs(&entryList, false),
 				cfgui.MemberOnly(),
 				cfgui.RecordFiles(),
+				cfgui.Avatars(),
 			},
 		},
 	}

cmd/slackdump/internal/archive/search.go

@@ -36,7 +36,7 @@ var CmdSearch = &base.Command{
 //go:embed assets/search.md
 var searchMD string
 
-const flagMask = cfg.OmitUserCacheFlag | cfg.OmitCacheDir | cfg.OmitTimeframeFlag | cfg.OmitMemberOnlyFlag
+const flagMask = cfg.OmitUserCacheFlag | cfg.OmitCacheDir | cfg.OmitTimeframeFlag | cfg.OmitMemberOnlyFlag | cfg.OmitDownloadAvatarsFlag
 
 var cmdSearchMessages = &base.Command{
 	UsageLine:   "slackdump search messages [flags] query terms",
@@ -151,7 +151,6 @@ func initController(ctx context.Context, args []string) (*control.Controller, fu
 		fsadapter.NewDirectory(cd.Name()),
 		lg,
 	)
-
 	pb := bootstrap.ProgressBar(ctx, lg, progressbar.OptionShowCount()) // progress bar
 	stop := func() {
 		dlstop()
@@ -175,7 +174,13 @@ func initController(ctx context.Context, args []string) (*control.Controller, fu
 	var (
 		subproc = fileproc.NewExport(fileproc.STmattermost, dl)
 		stream  = sess.Stream(sopts...)
-		ctrl    = control.New(cd, stream, control.WithLogger(lg), control.WithFiler(subproc), control.WithFlags(control.Flags{RecordFiles: cfg.RecordFiles}))
+		ctrl    = control.New(
+			cd,
+			stream,
+			control.WithLogger(lg),
+			control.WithFiler(subproc),
+			control.WithFlags(control.Flags{RecordFiles: cfg.RecordFiles}),
+		)
 	)
 	return ctrl, stop, nil
 }

cmd/slackdump/internal/cfg/cfg.go

@@ -41,9 +41,10 @@ var (
 	ForceEnterprise bool
 	MachineIDOvr    string // Machine ID override
 
-	MemberOnly    bool
-	DownloadFiles bool
-	RecordFiles   bool // record file chunks in chunk files.
+	MemberOnly      bool
+	DownloadFiles   bool
+	DownloadAvatars bool
+	RecordFiles     bool // record file chunks in chunk files.
 
 	// Oldest is the default timestamp of the oldest message to fetch, that is
 	// used by the dump and export commands.
@@ -91,6 +92,7 @@ const (
 	OmitChunkCacheFlag
 	OmitMemberOnlyFlag
 	OmitRecordFilesFlag
+	OmitDownloadAvatarsFlag
 
 	OmitAll = OmitConfigFlag |
 		OmitDownloadFlag |
@@ -102,7 +104,8 @@ const (
 		OmitTimeframeFlag |
 		OmitChunkCacheFlag |
 		OmitMemberOnlyFlag |
-		OmitRecordFilesFlag
+		OmitRecordFilesFlag |
+		OmitDownloadAvatarsFlag
 )
 
 // SetBaseFlags sets base flags
@@ -129,9 +132,12 @@ func SetBaseFlags(fs *flag.FlagSet, mask FlagMask) {
 	}
 	if mask&OmitDownloadFlag == 0 {
 		fs.BoolVar(&DownloadFiles, "files", true, "enables file attachments download (to disable, specify: -files=false)")
+		if mask&OmitRecordFilesFlag == 0 {
+			fs.BoolVar(&RecordFiles, "files-rec", false, "include file chunks in chunk files")
+		}
 	}
-	if mask&OmitRecordFilesFlag == 0 && mask&OmitDownloadFlag == 0 {
-		fs.BoolVar(&RecordFiles, "files-rec", false, "include file chunks in chunk files")
+	if mask&OmitDownloadAvatarsFlag == 0 {
+		fs.BoolVar(&DownloadAvatars, "avatars", true, "enables user avatar download (placed in __avatars directory)")
 	}
 	if mask&OmitConfigFlag == 0 {
 		fs.StringVar(&ConfigFile, "api-config", "", "configuration `file` with Slack API limits overrides.\nYou can generate one with default values with 'slackdump config new`")

cmd/slackdump/internal/convertcmd/convert.go

@@ -22,11 +22,13 @@ var CmdConvert = &base.Command{
 	Run:       runConvert,
 	UsageLine: "slackdump convert [flags] <source>",
 	Short:     "convert slackdump chunks to various formats",
-	Long: `
+	Long: `# Convert Command
+Convert slackdump archive format to various formats.
 
+Currently only "export" format is supported.
 `,
 	CustomFlags: false,
-	FlagMask:    cfg.OmitAll & ^cfg.OmitDownloadFlag &^ cfg.OmitOutputFlag,
+	FlagMask:    cfg.OmitAll & ^cfg.OmitDownloadFlag &^ cfg.OmitOutputFlag &^ cfg.OmitDownloadAvatarsFlag,
 	PrintFlags:  true,
 }
 
@@ -63,8 +65,9 @@ func runConvert(ctx context.Context, cmd *base.Command, args []string) error {
 	lg.InfoContext(ctx, "converting", "input_format", params.inputfmt, "source", args[0], "output_format", params.outputfmt, "output", cfg.Output)
 
 	cflg := convertflags{
-		withFiles: cfg.DownloadFiles,
-		stt:       params.storageType,
+		withFiles:   cfg.DownloadFiles,
+		withAvatars: cfg.DownloadAvatars,
+		stt:         params.storageType,
 	}
 	start := time.Now()
 	if err := fn(ctx, args[0], cfg.Output, cflg); err != nil {
@@ -96,8 +99,9 @@ var converters = map[datafmt]map[datafmt]convertFunc{
 }
 
 type convertflags struct {
-	withFiles bool
-	stt       fileproc.StorageType
+	withFiles   bool
+	withAvatars bool
+	stt         fileproc.StorageType
 }
 
 func chunk2export(ctx context.Context, src, trg string, cflg convertflags) error {
@@ -121,6 +125,8 @@ func chunk2export(ctx context.Context, src, trg string, cflg convertflags) error
 		cd,
 		fsa,
 		convert.WithIncludeFiles(cflg.withFiles),
+		convert.WithIncludeAvatars(cflg.withAvatars),
+		convert.WithSrcFileLoc(sttFn),
 		convert.WithTrgFileLoc(sttFn),
 		convert.WithLogger(cfg.Log),
 	)

cmd/slackdump/internal/dump/dump.go

@@ -38,7 +38,7 @@ var CmdDump = &base.Command{
 	Long:        dumpMd,
 	RequireAuth: true,
 	PrintFlags:  true,
-	FlagMask:    cfg.OmitMemberOnlyFlag | cfg.OmitRecordFilesFlag,
+	FlagMask:    cfg.OmitMemberOnlyFlag | cfg.OmitRecordFilesFlag | cfg.OmitDownloadAvatarsFlag,
 }
 
 func init() {

cmd/slackdump/internal/emoji/emoji.go

@@ -29,7 +29,7 @@ var CmdEmoji = &base.Command{
 	UsageLine:   "slackdump emoji [flags]",
 	Short:       "download workspace emoticons ಠ_ಠ",
 	Long:        emojiMD,
-	FlagMask:    cfg.OmitDownloadFlag | cfg.OmitConfigFlag | cfg.OmitChunkCacheFlag | cfg.OmitUserCacheFlag,
+	FlagMask:    cfg.OmitAll &^ cfg.OmitAuthFlags,
 	RequireAuth: true,
 	PrintFlags:  true,
 }

cmd/slackdump/internal/export/v3.go

@@ -56,6 +56,9 @@ func export(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, list
 	dlEnabled := cfg.DownloadFiles && params.ExportStorageType != fileproc.STnone
 	sdl, stop := fileproc.NewDownloader(ctx, dlEnabled, sess.Client(), fsa, lg)
 	defer stop()
+	avdl, avstop := fileproc.NewDownloader(ctx, cfg.DownloadAvatars, sess.Client(), fsa, lg)
+	defer avstop()
+	avp := fileproc.NewAvatarProc(avdl)
 
 	pb := bootstrap.ProgressBar(ctx, lg, progressbar.OptionShowCount()) // progress bar
 
@@ -81,6 +84,7 @@ func export(ctx context.Context, sess *slackdump.Session, fsa fsadapter.FS, list
 		control.WithLogger(lg),
 		control.WithFlags(flags),
 		control.WithTransformer(tf),
+		control.WithAvatarProcessor(avp),
 	)
 
 	lg.InfoContext(ctx, "running export...")

cmd/slackdump/internal/export/wizard.go

@@ -51,6 +51,7 @@ func (fl *exportFlags) configuration() cfgui.Configuration {
 						)),
 				},
 				cfgui.MemberOnly(),
+				cfgui.Avatars(),
 				{
 					Name:        "Export Token",
 					Value:       fl.ExportToken,

cmd/slackdump/internal/list/common.go

@@ -18,7 +18,7 @@ import (
 	"github.com/rusq/slackdump/v3/types"
 )
 
-const flagMask = cfg.OmitDownloadFlag | cfg.OmitMemberOnlyFlag
+const flagMask = cfg.OmitAll &^ cfg.OmitAuthFlags &^ cfg.OmitCacheDir
 
 // CmdList is the list command.  The logic is in the subcommands.
 var CmdList = &base.Command{

cmd/slackdump/internal/ui/cfgui/common_params.go

@@ -43,3 +43,12 @@ func RecordFiles() Parameter {
 		Updater:     updaters.NewBool(&cfg.RecordFiles),
 	}
 }
+
+func Avatars() Parameter {
+	return Parameter{
+		Name:        "Download Avatars",
+		Value:       Checkbox(cfg.DownloadAvatars),
+		Description: "Download avatars",
+		Updater:     updaters.NewBool(&cfg.DownloadAvatars),
+	}
+}

internal/chunk/control/control.go

@@ -35,6 +35,8 @@ type Controller struct {
 	// files subprocessor, if not configured with options, it's a noop, as
 	// it's not necessary for all use cases.
 	filer processor.Filer
+	// avp is avatar downloader
+	avp processor.Avatars
 	// lg is the logger
 	lg *slog.Logger
 	// flags
@@ -51,6 +53,13 @@ func WithFiler(f processor.Filer) Option {
 	}
 }
 
+// WithAvatarProcessor configures the controller with an avatar downloader.
+func WithAvatarProcessor(avp processor.Avatars) Option {
+	return func(c *Controller) {
+		c.avp = avp
+	}
+}
+
 // WithFlags configures the controller with flags.
 func WithFlags(f Flags) Option {
 	return func(c *Controller) {
@@ -83,6 +92,7 @@ func New(cd *chunk.Directory, s Streamer, opts ...Option) *Controller {
 		s:     s,
 		filer: &noopFiler{},
 		tf:    &noopTransformer{},
+		avp:   &noopAvatarProc{},
 		lg:    slog.Default(),
 	}
 	for _, opt := range opts {
@@ -167,7 +177,7 @@ func (c *Controller) Run(ctx context.Context, list *structures.EntityList) error
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			if err := userWorker(ctx, c.s, c.cd, c.tf); err != nil {
+			if err := userWorker(ctx, c.s, c.avp, c.cd, c.tf); err != nil {
 				errC <- Error{"user", "worker", err}
 				return
 			}

internal/chunk/control/noop.go

@@ -7,6 +7,7 @@ import (
 	"context"
 
 	"github.com/rusq/slack"
+
 	"github.com/rusq/slackdump/v3/internal/chunk"
 )
 
@@ -29,3 +30,9 @@ func (n *noopTransformer) Transform(ctx context.Context, id chunk.FileID) error
 func (n *noopTransformer) Wait() error {
 	return nil
 }
+
+type noopAvatarProc struct{}
+
+func (n *noopAvatarProc) Avatars(ctx context.Context, users []slack.User) error {
+	return nil
+}

internal/chunk/control/workers.go

@@ -10,16 +10,20 @@ import (
 	"github.com/rusq/slackdump/v3/internal/structures"
 
 	"github.com/rusq/slack"
+
 	"github.com/rusq/slackdump/v3/internal/chunk"
 	"github.com/rusq/slackdump/v3/internal/chunk/dirproc"
 	"github.com/rusq/slackdump/v3/internal/chunk/transform"
 	"github.com/rusq/slackdump/v3/processor"
 )
 
-func userWorker(ctx context.Context, s Streamer, chunkdir *chunk.Directory, tf TransformStarter) error {
-	var users = make([]slack.User, 0, 100)
+func userWorker(ctx context.Context, s Streamer, avp processor.Avatars, chunkdir *chunk.Directory, tf TransformStarter) error {
+	users := make([]slack.User, 0, 100)
 	userproc, err := dirproc.NewUsers(chunkdir, dirproc.WithUsers(func(us []slack.User) error {
 		users = append(users, us...)
+		if err := avp.Avatars(ctx, us); err != nil {
+			slog.Warn("error downloading avatars", "error", err)
+		}
 		return nil
 	}))
 	if err != nil {

internal/chunk/dirproc/users.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/rusq/slack"
+
 	"github.com/rusq/slackdump/v3/internal/chunk"
 	"github.com/rusq/slackdump/v3/processor"
 )

internal/chunk/transform/export.go

@@ -99,7 +99,7 @@ func (e *ExpConverter) writeMessages(ctx context.Context, pl *chunk.File, ci *sl
 	uidx := types.Users(e.users).IndexByID()
 	trgdir := ExportChanName(ci)
 
-	var mm []export.ExportMessage = make([]export.ExportMessage, 0, 100)
+	mm := make([]export.ExportMessage, 0, 100)
 	var prevDt string
 	var currDt string
 	if err := pl.Sorted(ctx, false, func(ts time.Time, m *slack.Message) error {
@@ -117,7 +117,7 @@ func (e *ExpConverter) writeMessages(ctx context.Context, pl *chunk.File, ci *sl
 		// the "thread" is only used to collect statistics.  Thread messages
 		// are passed by Sorted and written as a normal course of action.
 		var thread []slack.Message
-		if m.ThreadTimestamp == m.Timestamp && m.LatestReply != structures.LatestReplyNoReplies {
+		if structures.IsThreadStart(m) && m.LatestReply != structures.LatestReplyNoReplies {
 			// get the thread for the initial thread message only.
 			var err error
 			thread, err = pl.AllThreadMessages(ci.ID, m.ThreadTimestamp)

internal/chunk/transform/fileproc/avatar.go

@@ -0,0 +1,55 @@
+package fileproc
+
+import (
+	"context"
+	"path"
+	"path/filepath"
+
+	"github.com/rusq/slack"
+)
+
+type AvatarProc struct {
+	dl       Downloader
+	filepath func(u *slack.User) string
+}
+
+func NewAvatarProc(dl Downloader) AvatarProc {
+	return AvatarProc{
+		dl:       dl,
+		filepath: AvatarPath,
+	}
+}
+
+func (a AvatarProc) Avatars(ctx context.Context, users []slack.User) error {
+	for _, u := range users {
+		if u.Profile.ImageOriginal == "" {
+			// skip empty
+			continue
+		}
+		if err := a.dl.Download(a.filepath(&u), u.Profile.ImageOriginal); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func AvatarPath(u *slack.User) string {
+	filename := path.Base(u.Profile.ImageOriginal)
+	return filepath.Join(
+		"__avatars",
+		u.ID,
+		filename,
+	)
+}
+
+func nvl(s string, ss ...string) string {
+	if s != "" {
+		return s
+	}
+	for _, v := range ss {
+		if v != "" {
+			return v
+		}
+	}
+	return ""
+}