Skip to content

Commit

Permalink
AI Chat: show web sources that were used (normally via web search) to…
Browse files Browse the repository at this point in the history
… generate the response
  • Loading branch information
petemill committed Jan 17, 2025
1 parent 54ef40b commit 384b9bb
Show file tree
Hide file tree
Showing 15 changed files with 407 additions and 31 deletions.
8 changes: 8 additions & 0 deletions .storybook/web-common-mock/create_sanitized_image_url.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
// Copyright (c) 2025 The Brave Authors. All rights reserved.
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

export default function createSanitizedImageUrl(imageUrl: string) {
return imageUrl
}
4 changes: 4 additions & 0 deletions .storybook/webpack.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,10 @@ const pathMap = {
// place to look.
path.resolve(__dirname, 'chrome-resources-mock'),
basePathMap['chrome://resources']
],
'$web-common': [
path.resolve(__dirname, 'web-common-mock'),
basePathMap['$web-common']
]
}

Expand Down
2 changes: 2 additions & 0 deletions browser/ui/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ source_set("ui") {
"webui/brave_webui_source.h",
"webui/skus_internals_ui.cc",
"webui/skus_internals_ui.h",
"webui/untrusted_sanitized_image_source.cc",
"webui/untrusted_sanitized_image_source.h",
]

# It doesn't make sense to view the webcompat webui on iOS & Android.
Expand Down
24 changes: 23 additions & 1 deletion browser/ui/webui/ai_chat/ai_chat_untrusted_conversation_ui.cc
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@
#include <string>
#include <utility>

#include "base/memory/weak_ptr.h"
#include "base/strings/escape.h"
#include "brave/browser/ai_chat/ai_chat_service_factory.h"
#include "brave/browser/ai_chat/ai_chat_urls.h"
#include "brave/browser/ui/side_panel/ai_chat/ai_chat_side_panel_utils.h"
#include "brave/browser/ui/webui/ai_chat/ai_chat_ui.h"
#include "brave/browser/ui/webui/untrusted_sanitized_image_source.h"
#include "brave/components/ai_chat/core/browser/ai_chat_service.h"
#include "brave/components/ai_chat/core/browser/constants.h"
#include "brave/components/ai_chat/core/browser/conversation_handler.h"
Expand All @@ -21,13 +23,16 @@
#include "brave/components/ai_chat/resources/grit/ai_chat_ui_generated_map.h"
#include "brave/components/constants/webui_url_constants.h"
#include "brave/components/l10n/common/localization_util.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/browser/ui/webui/webui_util.h"
#include "components/grit/brave_components_resources.h"
#include "content/public/browser/render_frame_host.h"
#include "content/public/browser/url_data_source.h"
#include "content/public/browser/web_contents.h"
#include "content/public/browser/web_ui.h"
#include "content/public/browser/web_ui_data_source.h"
#include "content/public/common/url_constants.h"
#include "url/url_constants.h"

#if BUILDFLAG(IS_ANDROID)
#include "brave/browser/ui/android/ai_chat/brave_leo_settings_launcher_helper.h"
Expand Down Expand Up @@ -67,6 +72,16 @@ class UIHandler : public ai_chat::mojom::UntrustedUIHandler {
base::EscapeQueryParamValue(search_query, true)));
}

void OpenURLFromResponse(const GURL& url) override {
if (!web_ui_->GetRenderFrameHost()->HasTransientUserActivation()) {
return;
}
if (!url.is_valid() || !url.SchemeIs(url::kHttpsScheme)) {
return;
}
OpenURL(url);
}

void BindParentPage(mojo::PendingReceiver<ai_chat::mojom::ParentUIFrame>
parent_ui_frame_receiver) override {
// Route the receiver to the parent frame
Expand Down Expand Up @@ -108,6 +123,8 @@ class UIHandler : public ai_chat::mojom::UntrustedUIHandler {

raw_ptr<content::WebUI> web_ui_ = nullptr;
mojo::Receiver<ai_chat::mojom::UntrustedUIHandler> receiver_;

base::WeakPtrFactory<UIHandler> weak_ptr_factory_{this};
};

} // namespace
Expand Down Expand Up @@ -159,7 +176,8 @@ AIChatUntrustedConversationUI::AIChatUntrustedConversationUI(
"style-src 'self' 'unsafe-inline' chrome-untrusted://resources;");
source->OverrideContentSecurityPolicy(
network::mojom::CSPDirectiveName::ImgSrc,
"img-src 'self' blob: chrome-untrusted://resources;");
"img-src 'self' blob: chrome-untrusted://resources "
"chrome-untrusted://image;");
source->OverrideContentSecurityPolicy(
network::mojom::CSPDirectiveName::FontSrc,
"font-src 'self' chrome-untrusted://resources;");
Expand All @@ -168,6 +186,10 @@ AIChatUntrustedConversationUI::AIChatUntrustedConversationUI(
base::StringPrintf("frame-ancestors %s;", kAIChatUIURL));
source->OverrideContentSecurityPolicy(
network::mojom::CSPDirectiveName::TrustedTypes, "trusted-types default;");

Profile* profile = Profile::FromWebUI(web_ui);
content::URLDataSource::Add(
profile, std::make_unique<UntrustedSanitizedImageSource>(profile));
}

AIChatUntrustedConversationUI::~AIChatUntrustedConversationUI() = default;
Expand Down
35 changes: 35 additions & 0 deletions browser/ui/webui/untrusted_sanitized_image_source.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// Copyright (c) 2025 The Brave Authors. All rights reserved.
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

#include "brave/browser/ui/webui/untrusted_sanitized_image_source.h"

#include <string>
#include <utility>

#include "base/strings/strcat.h"
#include "chrome/common/webui_url_constants.h"

std::string UntrustedSanitizedImageSource::GetSource() {
return base::StrCat({content::kChromeUIUntrustedScheme,
url::kStandardSchemeSeparator,
chrome::kChromeUIImageHost, "/"});
}

void UntrustedSanitizedImageSource::StartDataRequest(
const GURL& url,
const content::WebContents::Getter& wc_getter,
content::URLDataSource::GotDataCallback callback) {
if (!url.is_valid() || !url.SchemeIs(content::kChromeUIUntrustedScheme)) {
std::move(callback).Run(nullptr);
return;
}

// Change scheme to ChromeUIScheme for base class
GURL::Replacements replacements;
replacements.SetSchemeStr(content::kChromeUIScheme);

SanitizedImageSource::StartDataRequest(url.ReplaceComponents(replacements),
wc_getter, std::move(callback));
}
31 changes: 31 additions & 0 deletions browser/ui/webui/untrusted_sanitized_image_source.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright (c) 2025 The Brave Authors. All rights reserved.
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

#ifndef BRAVE_BROWSER_UI_WEBUI_UNTRUSTED_SANITIZED_IMAGE_SOURCE_H_
#define BRAVE_BROWSER_UI_WEBUI_UNTRUSTED_SANITIZED_IMAGE_SOURCE_H_

#include <string>

#include "chrome/browser/ui/webui/sanitized_image_source.h"
#include "content/public/browser/web_contents.h"
#include "url/gurl.h"

// Uses SanitizedImageSource for chrome-untrusted:// WebUIs
class UntrustedSanitizedImageSource : public SanitizedImageSource {
public:
using SanitizedImageSource::SanitizedImageSource;
UntrustedSanitizedImageSource(const UntrustedSanitizedImageSource&) = delete;
UntrustedSanitizedImageSource& operator=(const SanitizedImageSource&) =
delete;

// SanitizedImageSource:
std::string GetSource() override;
void StartDataRequest(
const GURL& url,
const content::WebContents::Getter& wc_getter,
content::URLDataSource::GotDataCallback callback) override;
};

#endif // BRAVE_BROWSER_UI_WEBUI_UNTRUSTED_SANITIZED_IMAGE_SOURCE_H_
44 changes: 44 additions & 0 deletions components/ai_chat/core/browser/engine/conversation_api_client.cc
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ using ConversationEvent = ConversationAPIClient::ConversationEvent;
using ConversationEventType = ConversationAPIClient::ConversationEventType;

constexpr char kRemotePath[] = "v1/conversation";

constexpr char kAllowedWebSourceFaviconHost[] = "imgs.search.brave.com";

#if !defined(OFFICIAL_BUILD)
constexpr char kAIChatServerUrl[] = "ai-chat-server-url";
#endif
Expand Down Expand Up @@ -114,6 +117,47 @@ mojom::ConversationEntryEventPtr ParseResponseEvent(
}
return mojom::ConversationEntryEvent::NewSearchQueriesEvent(
std::move(event));
} else if (*type == "webSources") {
const base::Value::List* sources = response_event.FindList("sources");
if (!sources) {
return nullptr;
}
auto event = mojom::WebSourcesEvent::New();
for (auto& item : *sources) {
if (!item.is_dict()) {
continue;
}
const base::Value::Dict& source = item.GetDict();
const std::string* title = source.FindString("title");
const std::string* url = source.FindString("url");
const std::string* favicon_url = source.FindString("favicon");
if (!title || !url || !favicon_url) {
DVLOG(2) << "Missing required fields in web source event: "
<< item.DebugString();
continue;
}
GURL item_url(*url);
GURL item_favicon_url(*favicon_url);
if (!item_url.is_valid() || !item_favicon_url.is_valid()) {
DVLOG(2) << "Invalid URL in webSource event: " << item.DebugString();
continue;
}
// Validate favicon is private source
if (!item_favicon_url.SchemeIs(url::kHttpsScheme) ||
base::CompareCaseInsensitiveASCII(item_favicon_url.host_piece(),
kAllowedWebSourceFaviconHost) !=
0) {
DVLOG(2) << "webSource event contained disallowed host or scheme: "
<< item.DebugString();
continue;
}
event->sources.push_back(
mojom::WebSource::New(*title, item_url, item_favicon_url));
}
if (event->sources.empty()) {
return nullptr;
}
return mojom::ConversationEntryEvent::NewSourcesEvent(std::move(event));
} else if (*type == "conversationTitle") {
const std::string* title = response_event.FindString("title");
if (!title) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ TEST_F(ConversationAPIUnitTest, PerformRequest_PremiumHeaders) {
"__Secure-sku#brave-leo-premium=" + expected_crediential);
EXPECT_NE(headers.find("x-brave-key"), headers.end());

// Verify body contains events in expected json format
// Verify input body contains input events in expected json format
EXPECT_STREQ(GetEventsJson(body).c_str(),
FormatComparableEventsJson(expected_events_body).c_str());

Expand All @@ -254,8 +254,8 @@ TEST_F(ConversationAPIUnitTest, PerformRequest_PremiumHeaders) {
EXPECT_TRUE(selected_language.has_value());
EXPECT_TRUE(selected_language.value().empty());

// Send some event responses so that we can verify it is passed
// through to the PerformRequest callbacks.
// Send some event responses so that we can verify they are passed
// through to the PerformRequest callbacks as events.
{
base::Value result(base::Value::Type::DICT);
result.GetDict().Set("type", "isSearching");
Expand All @@ -270,6 +270,50 @@ TEST_F(ConversationAPIUnitTest, PerformRequest_PremiumHeaders) {
result.GetDict().Set("queries", std::move(queries));
data_received_callback.Run(base::ok(std::move(result)));
}
{
base::Value result(base::Value::Type::DICT);
result.GetDict().Set("type", "webSources");
base::Value sources(base::Value::Type::LIST);
{
// Invalid because it doesn't contain the expected host
base::Value query(base::Value::Type::DICT);
query.GetDict().Set("title", "Star Wars");
query.GetDict().Set("url", "https://starwars.com");
query.GetDict().Set("favicon", "https://starwars.com/favicon");
sources.GetList().Append(std::move(query));
}
{
// Invalid because it doesn't contain the expected scheme
base::Value query(base::Value::Type::DICT);
query.GetDict().Set("title", "Star Wars");
query.GetDict().Set("url", "https://starwars.com");
query.GetDict().Set(
"favicon", "http://imgs.search.brave.com/starwars.com/favicon");
sources.GetList().Append(std::move(query));
}
{
// Valid
base::Value query(base::Value::Type::DICT);
query.GetDict().Set("title", "Star Wars");
query.GetDict().Set("url", "https://starwars.com");
query.GetDict().Set(
"favicon",
"https://imgs.search.brave.com/starwars.com/favicon");
sources.GetList().Append(std::move(query));
}
{
// Valid
base::Value query(base::Value::Type::DICT);
query.GetDict().Set("title", "Star Trek");
query.GetDict().Set("url", "https://startrek.com");
query.GetDict().Set(
"favicon",
"https://imgs.search.brave.com/startrek.com/favicon");
sources.GetList().Append(std::move(query));
}
result.GetDict().Set("sources", std::move(sources));
data_received_callback.Run(base::ok(std::move(result)));
}
{
base::Value result(base::Value::Type::DICT);
result.GetDict().Set("type", "completion");
Expand Down Expand Up @@ -308,6 +352,21 @@ TEST_F(ConversationAPIUnitTest, PerformRequest_PremiumHeaders) {
EXPECT_EQ(queries[0], "Star Wars");
EXPECT_EQ(queries[1], "Star Trek");
});
EXPECT_CALL(mock_callbacks, OnDataReceived(_))
.InSequence(seq)
.WillOnce([&](mojom::ConversationEntryEventPtr event) {
EXPECT_TRUE(event->is_sources_event());
auto& sources = event->get_sources_event()->sources;
EXPECT_EQ(sources.size(), 2u);
EXPECT_EQ(sources[0]->title, "Star Wars");
EXPECT_EQ(sources[1]->title, "Star Trek");
EXPECT_EQ(sources[0]->url.spec(), "https://starwars.com/");
EXPECT_EQ(sources[1]->url.spec(), "https://startrek.com/");
EXPECT_EQ(sources[0]->favicon_url.spec(),
"https://imgs.search.brave.com/starwars.com/favicon");
EXPECT_EQ(sources[1]->favicon_url.spec(),
"https://imgs.search.brave.com/startrek.com/favicon");
});
EXPECT_CALL(mock_callbacks, OnDataReceived(_))
.InSequence(seq)
.WillOnce([&](mojom::ConversationEntryEventPtr event) {
Expand Down
11 changes: 11 additions & 0 deletions components/ai_chat/core/common/mojom/ai_chat.mojom
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,16 @@ struct SearchStatusEvent {
bool is_searching = true;
};

struct WebSource {
string title;
url.mojom.Url url;
url.mojom.Url favicon_url;
};

struct WebSourcesEvent {
array<WebSource> sources;
};

struct CompletionEvent {
string completion;
};
Expand All @@ -181,6 +191,7 @@ union ConversationEntryEvent {
PageContentRefineEvent page_content_refine_event;
SearchQueriesEvent search_queries_event;
SearchStatusEvent search_status_event;
WebSourcesEvent sources_event;

// These events don't normally get added to the conversation entry
// but are used in engine responses.
Expand Down
9 changes: 9 additions & 0 deletions components/ai_chat/core/common/mojom/untrusted_frame.mojom
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

module ai_chat.mojom;

import "url/mojom/url.mojom";

// Interfaces for communication between the untrusted content frame and both
// the Browser and the parent trusted UI frame.

Expand All @@ -25,7 +27,14 @@ interface ParentUIFrame {
// Browser-side handler for untrusted frame that handles rendering of
// conversation entries.
interface UntrustedUIHandler {
// Open a URL for the web search query the assistant used for a response
OpenSearchURL(string query);

OpenLearnMoreAboutBraveSearchWithLeo();

// Opens a URL linked from an assistant response or web search event in the
// response. Could be any http/https destination.
OpenURLFromResponse(url.mojom.Url url);

BindParentPage(pending_receiver<ParentUIFrame> parent_frame);
};
Loading

0 comments on commit 384b9bb

Please sign in to comment.