Skip to content

Commit

Permalink
fix: download from s3 custom domain
Browse files Browse the repository at this point in the history
  • Loading branch information
lihsai0 committed Nov 21, 2024
1 parent 5fdc278 commit 659e720
Show file tree
Hide file tree
Showing 8 changed files with 73 additions and 24 deletions.
3 changes: 2 additions & 1 deletion src/common/ipc-actions/download.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog";
import {Domain} from "kodo-s3-adapter-sdk/dist/adapter";
import {Domain, UrlStyle} from "kodo-s3-adapter-sdk/dist/adapter";

import {ClientOptionsSerialized} from "@common/qiniu";
import {Status} from "@common/models/job/types";
Expand All @@ -24,6 +24,7 @@ export interface DownloadOptions {
bucket: string,

domain?: Domain,
urlStyle?: UrlStyle,

isOverwrite: boolean,
storageClasses: StorageClass[],
Expand Down
12 changes: 10 additions & 2 deletions src/common/models/job/download-job.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import fs, {constants as fsConstants, promises as fsPromises} from "fs";

import lodash from "lodash";
import {Downloader} from "kodo-s3-adapter-sdk";
import {Adapter, Domain, ObjectHeader, StorageClass} from "kodo-s3-adapter-sdk/dist/adapter";
import {Adapter, Domain, ObjectHeader, StorageClass, UrlStyle} from "kodo-s3-adapter-sdk/dist/adapter";
import {NatureLanguage} from "kodo-s3-adapter-sdk/dist/uplog";

import {ClientOptions, createQiniuClient} from "@common/qiniu";
Expand Down Expand Up @@ -36,7 +36,8 @@ interface DownloadOptions {

interface OptionalOptions extends DownloadOptions {
id: string,
domain?: Domain,
domain: Domain | undefined,
urlStyle: UrlStyle | undefined,

status: Status,
message: string,
Expand All @@ -58,6 +59,8 @@ type Options = RequiredOptions & Partial<OptionalOptions>

const DEFAULT_OPTIONS: OptionalOptions = {
id: "",
domain: undefined,
urlStyle: undefined,

multipartDownloadThreshold: 100,
multipartDownloadSize: 8,
Expand Down Expand Up @@ -86,6 +89,7 @@ type PersistInfo = {
from: RequiredOptions["from"],
backendMode: RequiredOptions["clientOptions"]["backendMode"],
domain: OptionalOptions["domain"],
urlStyle: OptionalOptions["urlStyle"],
prog: OptionalOptions["prog"],
status: Exclude<OptionalOptions["status"], Status.Waiting | Status.Running>,
message: OptionalOptions["message"],
Expand All @@ -112,6 +116,8 @@ export default class DownloadJob extends TransferJob {
): DownloadJob {
return new DownloadJob({
id,
domain: persistInfo.domain,
urlStyle: persistInfo.urlStyle,
status: persistInfo.status,
message: persistInfo.message,

Expand Down Expand Up @@ -213,6 +219,7 @@ export default class DownloadJob extends TransferJob {
to: this.options.to,
from: this.options.from,
domain: this.options.domain,
urlStyle: this.options.urlStyle,
storageClasses: this.options.storageClasses,
backendMode: this.options.clientOptions.backendMode,

Expand Down Expand Up @@ -297,6 +304,7 @@ export default class DownloadJob extends TransferJob {
this.tempFilePath,
this.options.domain,
{
urlStyle: this.options.urlStyle,
recoveredFrom: this.prog.resumable
? this.prog.loaded
: 0,
Expand Down
1 change: 1 addition & 0 deletions src/main/transfer-managers/download-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@ export default class DownloadManager extends TransferManager<DownloadJob, Config
overwrite: downloadOptions.isOverwrite,
region: downloadOptions.region,
domain: downloadOptions.domain,
urlStyle: downloadOptions.urlStyle,

multipartDownloadThreshold: this.config.multipartThreshold,
multipartDownloadSize: this.config.multipartSize,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ import {BackendMode} from "@common/qiniu"

import {useI18n} from "@renderer/modules/i18n";
import {useAuth} from "@renderer/modules/auth";
import {getContent, saveContent} from "@renderer/modules/qiniu-client";
import {getContent, saveContent, getStyleForSignature} from "@renderer/modules/qiniu-client";
import {DomainAdapter, NON_OWNED_DOMAIN} from "@renderer/modules/qiniu-client-hooks";
import {DiffView, EditorView} from "@renderer/modules/codemirror";
import {useFileOperation} from "@renderer/modules/file-operation";

import LoadingHolder from "@renderer/components/loading-holder";

Expand All @@ -34,6 +35,7 @@ const CodeContent: React.FC<CodeContentProps> = ({
}) => {
const {translate} = useI18n();
const {currentUser} = useAuth();
const {bucketPreferBackendMode: preferBackendMode} = useFileOperation();

// code mirror editor
const editorRef = useRef<Editor>();
Expand Down Expand Up @@ -76,13 +78,19 @@ const CodeContent: React.FC<CodeContentProps> = ({
endpointType: currentUser.endpointType,
preferS3Adapter: domain.apiScope === BackendMode.S3,
};
const domainToGet = domain?.name === NON_OWNED_DOMAIN.name
? undefined
: domain;
getContent(
regionId,
bucketName,
filePath,
domain.name === NON_OWNED_DOMAIN.name
? undefined
: domain,
domainToGet,
getStyleForSignature({
domain: domainToGet,
preferBackendMode,
currentEndpointType: currentUser.endpointType,
}),
opt,
)
.then(contentBuffer => {
Expand Down
12 changes: 9 additions & 3 deletions src/renderer/modules/qiniu-client/common.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,16 @@ import * as MockAuth from "@common/qiniu/_mock-helpers_/auth";

import { RegionService } from "kodo-s3-adapter-sdk/dist/region_service";

jest.mock("kodo-s3-adapter-sdk/dist/region_service", () => ({
jest.mock("kodo-s3-adapter-sdk/dist/region_service", () => {
class MockedRegionService extends jest.fn() {
static clearCache = jest.fn();
clearCache = jest.fn();
}
return {
__esModule: true,
RegionService: jest.fn().mockReturnValue({ clearCache: () => {} }),
}));
RegionService: MockedRegionService,
}
});

import { Kodo as KodoAdapter } from "kodo-s3-adapter-sdk/dist/kodo";
import { S3 as S3Adapter } from "kodo-s3-adapter-sdk/dist/s3";
Expand Down
6 changes: 4 additions & 2 deletions src/renderer/modules/qiniu-client/files.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ import {
ObjectInfo,
PartialObjectError,
StorageClass,
StorageObject
StorageObject,
UrlStyle,
} from "kodo-s3-adapter-sdk/dist/adapter";

import Duration from "@common/const/duration";
Expand Down Expand Up @@ -443,6 +444,7 @@ describe("test qiniu-client/files.ts", () => {
"bucket-kodo-browser-Kodo-getContent",
qiniuPathConvertor.fromQiniuPath("qiniu-client/file-to-get"),
mockDomain,
undefined,
mockOpt,
);
expect(QiniuClientCommon.getDefaultClient).toBeCalledTimes(1);
Expand Down Expand Up @@ -607,7 +609,7 @@ describe("test qiniu-client/files.ts", () => {
mockDataKey,
undefined,
10,
name === "S3" ? "path" : "bucketEndpoint",
name === "S3" ? UrlStyle.Path : UrlStyle.BucketEndpoint,
mockOpt,
);
expect(QiniuClientCommon.getDefaultClient).toBeCalledTimes(1);
Expand Down
28 changes: 20 additions & 8 deletions src/renderer/modules/qiniu-client/files.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
import * as qiniuPathConvertor from "qiniu-path/dist/src/convert";
import { Path as QiniuPath } from "qiniu-path/dist/src/path";
import { Adapter, Domain, FrozenInfo, ObjectInfo, PartialObjectError, StorageClass, TransferObject } from 'kodo-s3-adapter-sdk/dist/adapter'
import {
Adapter,
Domain,
FrozenInfo,
ObjectInfo,
PartialObjectError,
StorageClass,
TransferObject,
UrlStyle,
} from 'kodo-s3-adapter-sdk/dist/adapter'

import {BackendMode} from "@common/qiniu";
import Duration from "@common/const/duration";
Expand Down Expand Up @@ -213,6 +222,7 @@ export async function getContent(
bucket: string,
key: QiniuPath | string,
domain: Domain | undefined,
style: UrlStyle | undefined,
opt: GetAdapterOptionParam,
):Promise<Buffer> {
return await getDefaultClient(opt).enter("getContent", async client => {
Expand All @@ -223,6 +233,7 @@ export async function getContent(
key: key.toString(),
},
domain,
style,
);
return obj.data;
}, {
Expand Down Expand Up @@ -333,14 +344,16 @@ export function getStyleForSignature({
domain: Domain | undefined,
preferBackendMode: BackendMode | undefined,
currentEndpointType: EndpointType,
}): "path" | "virtualHost" | "bucketEndpoint" {
}): UrlStyle {
if (domain?.apiScope === BackendMode.S3) {
return "bucketEndpoint";
return UrlStyle.BucketEndpoint;
} else if (!domain || preferBackendMode === BackendMode.S3) {
// some private cloud not support wildcard domain name resolving
return currentEndpointType === EndpointType.Public ? "virtualHost" : "path";
return currentEndpointType === EndpointType.Public
? UrlStyle.VirtualHost
: UrlStyle.Path;
} else {
return "bucketEndpoint";
return UrlStyle.BucketEndpoint;
}
}

Expand All @@ -350,7 +363,7 @@ export async function signatureUrl(
key: QiniuPath | string,
domain: Domain | undefined,
expires: number, // seconds
style: "path" | "virtualHost" | "bucketEndpoint",
style: UrlStyle,
opt: GetAdapterOptionParam,
): Promise<URL> {
const deadline = new Date();
Expand Down Expand Up @@ -909,7 +922,7 @@ export async function signatureUrls(
items: FileItem.Item[],
domain: Domain | undefined,
expires: number,
style: "path" | "virtualHost" | "bucketEndpoint",
style: UrlStyle,
progressFn: (progress: Progress) => void,
onEachSuccess: (file: FileItem.File, url: URL) => void,
onError: (err: any) => void,
Expand Down Expand Up @@ -969,4 +982,3 @@ export async function signatureUrls(
return results;
}
}

19 changes: 15 additions & 4 deletions src/renderer/pages/browse/files/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ import {useAuth} from "@renderer/modules/auth";
import {KodoNavigator, useKodoNavigator} from "@renderer/modules/kodo-address";
import {ContentViewStyle, appPreferences, useEndpointConfig} from "@renderer/modules/user-config-store";

import {BucketItem, FileItem, isAccelerateUploadingAvailable} from "@renderer/modules/qiniu-client";
import {DomainAdapter, useLoadDomains, useLoadFiles} from "@renderer/modules/qiniu-client-hooks";
import {BucketItem, FileItem, getStyleForSignature, isAccelerateUploadingAvailable} from "@renderer/modules/qiniu-client";
import {DomainAdapter, NON_OWNED_DOMAIN, useLoadDomains, useLoadFiles} from "@renderer/modules/qiniu-client-hooks";
import ipcDownloadManager from "@renderer/modules/electron-ipc-manages/ipc-download-manager";
import * as AuditLog from "@renderer/modules/audit-log";
import {useFileOperation} from "@renderer/modules/file-operation";
Expand All @@ -38,7 +38,10 @@ interface FilesProps {
const Files: React.FC<FilesProps> = (props) => {
const {currentLanguage, translate} = useI18n();
const {currentUser, shareSession} = useAuth();
const {bucketGrantedPermission} = useFileOperation();
const {
bucketPreferBackendMode: preferBackendMode,
bucketGrantedPermission,
} = useFileOperation();

const {
state: appPreferencesState,
Expand Down Expand Up @@ -350,13 +353,21 @@ const Files: React.FC<FilesProps> = (props) => {
to: destDirectoryPath,
from: remoteObjects.map(i => i.key),
});
const domain = selectedDomain.name === NON_OWNED_DOMAIN.name
? undefined
: selectedDomain;
ipcDownloadManager.addJobs({
remoteObjects,
destPath: destDirectoryPath,
downloadOptions: {
region: currentRegion.s3Id,
bucket: currentBucket.name,
domain: selectedDomain,
domain: domain,
urlStyle: getStyleForSignature({
domain: domain,
preferBackendMode,
currentEndpointType: currentUser.endpointType,
}),
isOverwrite: appPreferencesData.overwriteDownloadEnabled,
storageClasses: currentRegion.storageClasses,
// userNatureLanguage needs mid-dash but i18n using lo_dash
Expand Down

0 comments on commit 659e720

Please sign in to comment.