Skip to content

Commit 2c79855

Browse files
cmrigneycmrigney
committed
Hacky to fix page imports by using markdown + not serializing into Go.
1 parent 4da6364 commit 2c79855

9 files changed

Lines changed: 194 additions & 135 deletions

File tree

‎internal/agent-task/convert-page-task.go‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ type ConvertPageTask struct {
2828
firecrawlScraper scraper.Scraper
2929
payloadCMSClient *payloadcms.Client
3030
llm provider.Provider
31-
pageCache pagecache.PageCache
31+
htmlCache pagecache.PageCache
32+
markdownCache pagecache.PageCache
3233
}
3334

3435
func NewConvertPageTask(url string, pageID string, firecrawlScraper scraper.Scraper, payloadCMSClient *payloadcms.Client, llm provider.Provider) *ConvertPageTask {
@@ -39,7 +40,8 @@ func NewConvertPageTask(url string, pageID string, firecrawlScraper scraper.Scra
3940
firecrawlScraper: firecrawlScraper,
4041
payloadCMSClient: payloadCMSClient,
4142
llm: llm,
42-
pageCache: pagecache.NewPageCache("html"),
43+
htmlCache: pagecache.NewPageCache("html"),
44+
markdownCache: pagecache.NewPageCache("md"),
4345
}
4446
}
4547

@@ -57,22 +59,28 @@ func (t *ConvertPageTask) Execute(ctx context.Context) error {
5759
defer cancel()
5860

5961
var html string
60-
cachedPage, err := t.pageCache.GetCachedPage(t.url)
62+
var markdown string
63+
cachedPage, err := t.htmlCache.GetCachedPage(t.url)
64+
cachedMarkdown, err := t.markdownCache.GetCachedPage(t.url)
6165
if err != nil {
6266
return fmt.Errorf("error getting cached page: %w", err)
6367
}
64-
if cachedPage != "" {
68+
if cachedPage != "" && cachedMarkdown != "" {
6569
html = cachedPage
70+
markdown = cachedMarkdown
6671
log.Println("[ConvertPageTask] Using cached page")
6772
} else {
6873
log.Println("[ConvertPageTask] No cached page found, scraping page")
69-
html, err = t.scrapePageHtml(ctx)
74+
html, markdown, err = t.scrapePage(ctx)
7075
if err != nil {
7176
return fmt.Errorf("error scraping page HTML: %w", err)
7277
}
73-
if err := t.pageCache.SetCachedPage(t.url, html); err != nil {
78+
if err := t.htmlCache.SetCachedPage(t.url, html); err != nil {
7479
return fmt.Errorf("error caching page: %w", err)
7580
}
81+
if err := t.markdownCache.SetCachedPage(t.url, markdown); err != nil {
82+
return fmt.Errorf("error caching markdown: %w", err)
83+
}
7684
}
7785

7886
convertPagePrompt, err := prompt.GetConvertPagePrompt()
@@ -100,7 +108,7 @@ func (t *ConvertPageTask) Execute(ctx context.Context) error {
100108
return fmt.Errorf("error creating runtime: %w", err)
101109
}
102110

103-
p := "I retrieved the following HTML from a church website at " + t.url + "\n\n" + html
111+
p := "The following markdown came from a church website at " + t.url + "\n\n" + markdown + "\n\n"
104112
sess := session.New(session.WithUserMessage("", p))
105113
sess.ToolsApproved = true
106114

@@ -113,7 +121,7 @@ func (t *ConvertPageTask) Execute(ctx context.Context) error {
113121
return nil
114122
}
115123

116-
func (t *ConvertPageTask) scrapePageHtml(ctx context.Context) (string, error) {
124+
func (t *ConvertPageTask) scrapePage(ctx context.Context) (string, string, error) {
117125
log.Println("[ConvertPageTask] Scraping page at", t.url)
118126
resultCh := t.firecrawlScraper.Scrape(t.url)
119127

@@ -122,15 +130,15 @@ func (t *ConvertPageTask) scrapePageHtml(ctx context.Context) (string, error) {
122130
case result = <-resultCh:
123131
break
124132
case <-ctx.Done():
125-
return "", ctx.Err()
133+
return "", "", ctx.Err()
126134
}
127135

128136
if result.Error != nil {
129137
log.Println("[ConvertPageTask] Error scraping page:", result.Error)
130-
return "", result.Error
138+
return "", "", result.Error
131139
}
132140

133-
return result.Html, nil
141+
return result.Html, result.Markdown, nil
134142
}
135143

136144
func downloadMedia(ctx context.Context, url string) ([]byte, error) {

‎internal/agent-task/tools.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ func toolExportPage(logTask string, pageID string, payloadCMSClient *payloadcms.
8989

9090
log.Println("[" + logTask + "] Patching page produced by agent")
9191

92-
if err := payloadCMSClient.UpdatePage(ctx, pageData); err != nil {
92+
if err := payloadCMSClient.UpdatePageRaw(ctx, p.PageJSON, pageID); err != nil {
9393
log.Println("["+logTask+"] Error patching page:", err)
9494
return nil, err
9595
}

‎internal/payloadcms/client.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,32 @@ func (c *Client) CreatePage(ctx context.Context, title string, slug string) (str
122122
return response.Doc.ID, nil
123123
}
124124

125+
func (c *Client) UpdatePageRaw(ctx context.Context, pageContent string, pageId string) error {
126+
req, err := http.NewRequestWithContext(ctx, "PATCH", c.cfg.BaseURL+"/api/pages/"+pageId, bytes.NewBuffer([]byte(pageContent)))
127+
if err != nil {
128+
return err
129+
}
130+
131+
req.Header.Set("Content-Type", "application/json")
132+
req.Header.Set("Authorization", "users API-Key "+c.cfg.APIKey)
133+
134+
resp, err := c.client.Do(req)
135+
if err != nil {
136+
return err
137+
}
138+
defer resp.Body.Close()
139+
140+
var response Response
141+
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
142+
return err
143+
}
144+
if len(response.Errors) > 0 {
145+
return response.Errors
146+
}
147+
148+
return nil
149+
}
150+
125151
func (c *Client) UpdatePage(ctx context.Context, page PagePatch) error {
126152
jsonBody, err := json.Marshal(page)
127153
if err != nil {

‎internal/payloadcms/page.go‎

Lines changed: 52 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ type PagePatch struct {
2626

2727
// Hero represents the hero section of a page
2828
type Hero struct {
29-
Type string `json:"type"` // 'none' | 'highImpact' | 'mediumImpact' | 'lowImpact'
30-
RichText *RichText `json:"richText,omitempty"`
31-
Links []HeroLink `json:"links,omitempty"`
32-
Media interface{} `json:"media,omitempty"` // string | Media
29+
Type string `json:"type"` // 'none' | 'highImpact' | 'mediumImpact' | 'lowImpact'
30+
RichText *RichText `json:"richText,omitempty"`
31+
Links []HeroLink `json:"links,omitempty"`
32+
Media string `json:"media,omitempty"` // string | Media
3333
}
3434

3535
// HeroLink represents a link in the hero section
@@ -71,9 +71,9 @@ type RichTextRoot struct {
7171

7272
// Meta represents metadata for pages and posts
7373
type Meta struct {
74-
Title *string `json:"title,omitempty"`
75-
Image interface{} `json:"image,omitempty"` // string | Media
76-
Description *string `json:"description,omitempty"`
74+
Title *string `json:"title,omitempty"`
75+
Image string `json:"image,omitempty"` // string | Media
76+
Description *string `json:"description,omitempty"`
7777
}
7878

7979
// Block represents a layout block (union type)
@@ -94,7 +94,7 @@ type Block struct {
9494
type Post struct {
9595
ID string `json:"id,omitempty"`
9696
Title string `json:"title"`
97-
HeroImage interface{} `json:"heroImage,omitempty"` // string | Media
97+
HeroImage string `json:"heroImage,omitempty"` // string | Media
9898
Content RichText `json:"content"`
9999
RelatedPosts []interface{} `json:"relatedPosts,omitempty"` // []string | []Post
100100
Series interface{} `json:"series,omitempty"` // string | Series
@@ -112,21 +112,21 @@ type Post struct {
112112

113113
// Event represents an event in the CMS
114114
type Event struct {
115-
ID string `json:"id,omitempty"`
116-
Title string `json:"title"`
117-
EventImage interface{} `json:"eventImage,omitempty"` // string | Media
118-
VideoLink *string `json:"videoLink,omitempty"`
119-
Content RichText `json:"content"`
120-
Location *string `json:"location,omitempty"`
121-
StartTime *string `json:"startTime,omitempty"`
122-
EndTime *string `json:"endTime,omitempty"`
123-
Meta *Meta `json:"meta,omitempty"`
124-
PublishedAt *string `json:"publishedAt,omitempty"`
125-
Slug *string `json:"slug,omitempty"`
126-
SlugLock *bool `json:"slugLock,omitempty"`
127-
UpdatedAt string `json:"updatedAt"`
128-
CreatedAt string `json:"createdAt"`
129-
Status *string `json:"_status,omitempty"`
115+
ID string `json:"id,omitempty"`
116+
Title string `json:"title"`
117+
EventImage string `json:"eventImage,omitempty"` // string | Media
118+
VideoLink *string `json:"videoLink,omitempty"`
119+
Content RichText `json:"content"`
120+
Location *string `json:"location,omitempty"`
121+
StartTime *string `json:"startTime,omitempty"`
122+
EndTime *string `json:"endTime,omitempty"`
123+
Meta *Meta `json:"meta,omitempty"`
124+
PublishedAt *string `json:"publishedAt,omitempty"`
125+
Slug *string `json:"slug,omitempty"`
126+
SlugLock *bool `json:"slugLock,omitempty"`
127+
UpdatedAt string `json:"updatedAt"`
128+
CreatedAt string `json:"createdAt"`
129+
Status *string `json:"_status,omitempty"`
130130
}
131131

132132
// PopulatedAuthor represents a populated author reference
@@ -177,14 +177,14 @@ type MediaSize struct {
177177

178178
// Series represents a series of posts
179179
type Series struct {
180-
ID string `json:"id,omitempty"`
181-
Title string `json:"title"`
182-
Image interface{} `json:"image,omitempty"` // string | Media
183-
Description string `json:"description"`
184-
Slug *string `json:"slug,omitempty"`
185-
SlugLock *bool `json:"slugLock,omitempty"`
186-
UpdatedAt string `json:"updatedAt"`
187-
CreatedAt string `json:"createdAt"`
180+
ID string `json:"id,omitempty"`
181+
Title string `json:"title"`
182+
Image string `json:"image,omitempty"` // string | Media
183+
Description string `json:"description"`
184+
Slug *string `json:"slug,omitempty"`
185+
SlugLock *bool `json:"slugLock,omitempty"`
186+
UpdatedAt string `json:"updatedAt"`
187+
CreatedAt string `json:"createdAt"`
188188
}
189189

190190
// Category represents a content category
@@ -233,27 +233,27 @@ type UserSession struct {
233233

234234
// TwoColumn represents a two-column layout block
235235
type TwoColumn struct {
236-
ImagePosition *string `json:"imagePosition,omitempty"` // 'left' | 'right'
237-
ImagePositionOnMobile *string `json:"imagePositionOnMobile,omitempty"` // 'top' | 'bottom'
238-
RichText *RichText `json:"richText,omitempty"`
239-
CenterTextOnMobile *bool `json:"centerTextOnMobile,omitempty"`
240-
SectionColor *string `json:"sectionColor,omitempty"` // 'none' | 'accent' | 'secondary' | 'dark'
241-
EnableLink *bool `json:"enableLink,omitempty"`
242-
Link *Link `json:"link,omitempty"`
243-
Image interface{} `json:"image,omitempty"` // string | Media
244-
ID *string `json:"id,omitempty"`
245-
BlockName *string `json:"blockName,omitempty"`
246-
BlockType string `json:"blockType"` // 'twoColumn'
236+
ImagePosition *string `json:"imagePosition,omitempty"` // 'left' | 'right'
237+
ImagePositionOnMobile *string `json:"imagePositionOnMobile,omitempty"` // 'top' | 'bottom'
238+
Markdown *string `json:"markdown,omitempty"`
239+
CenterTextOnMobile *bool `json:"centerTextOnMobile,omitempty"`
240+
SectionColor *string `json:"sectionColor,omitempty"` // 'none' | 'accent' | 'secondary' | 'dark'
241+
EnableLink *bool `json:"enableLink,omitempty"`
242+
Link *Link `json:"link,omitempty"`
243+
Image string `json:"image,omitempty"` // string | Media
244+
ID *string `json:"id,omitempty"`
245+
BlockName *string `json:"blockName,omitempty"`
246+
BlockType string `json:"blockType"` // 'twoColumn'
247247
}
248248

249249
// ImageBanner represents a image banner layout block
250250
type ImageBanner struct {
251-
RichText *RichText `json:"richText,omitempty"`
252-
Image interface{} `json:"image,omitempty"` // string | Media
253-
Links []Link `json:"links,omitempty"`
254-
ID *string `json:"id,omitempty"`
255-
BlockName *string `json:"blockName,omitempty"`
256-
BlockType string `json:"blockType"` // 'imageBanner'
251+
RichText *RichText `json:"richText,omitempty"`
252+
Image string `json:"image,omitempty"` // string | Media
253+
Links []Link `json:"links,omitempty"`
254+
ID *string `json:"id,omitempty"`
255+
BlockName *string `json:"blockName,omitempty"`
256+
BlockType string `json:"blockType"` // 'imageBanner'
257257
}
258258

259259
// CallToActionBlock represents a call-to-action block
@@ -284,10 +284,10 @@ type ContentColumn struct {
284284

285285
// MediaBlock represents a media block
286286
type MediaBlock struct {
287-
Media interface{} `json:"media"` // string | Media
288-
ID *string `json:"id,omitempty"`
289-
BlockName *string `json:"blockName,omitempty"`
290-
BlockType string `json:"blockType"` // 'mediaBlock'
287+
Media string `json:"media"` // string | Media
288+
ID *string `json:"id,omitempty"`
289+
BlockName *string `json:"blockName,omitempty"`
290+
BlockType string `json:"blockType"` // 'mediaBlock'
291291
}
292292

293293
// PostListBlock represents a post list block

‎internal/prompt/prompts/convert-page.md‎

Lines changed: 18 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,32 @@
1-
You are a specialized agent for converting church website HTML into structured data that can be used by PayloadCMS, a content management system.
1+
You are a specialized agent for converting church website markdown into structured data that can be used by PayloadCMS, a content management system.
22

33
Your primary responsibilities:
4-
1. Analyze HTML content provided by the user
5-
2. Discover and upload all media that will need to be referenced in the data structure using the `upload-media` tool
6-
3. Extract meaningful data and structure it according to the TypeScript type definition below
7-
4. Convert the HTML content into a JSON object that matches the required data structure
8-
5. Use the `export-page` tool to save the converted JSON output
4+
1. Upload all images referenced inside the markdown using the `upload-media` tool.
5+
2. Convert the markdown content into a JSON object that matches the required data structure
6+
3. Use the `export-page` tool to save the converted JSON output
97

108
Key guidelines:
11-
- Pay special attention to semantic meaning of content, not just HTML structure
12-
- Give it a modern beautiful layout within the boundaries of the type definition
13-
- Preserve important metadata like dates, times, contact information
14-
- Handle common church website elements: service times, pastor information, upcoming events, sermon series, ministry descriptions
15-
- Clean up and normalize text content (remove excessive whitespace, fix formatting)
16-
- Maintain hierarchical relationships in the data structure
17-
- Ignore navigation, just focus on page content
18-
- Validate that the output JSON conforms to the TypeScript type before exporting
9+
- Give it a modern beautiful layout
10+
- Make use of all the images as media as much as possible in the exported page
11+
- Preserve text content in the markdown as you translate it to the TypeScript type
1912

20-
Handling page content:
21-
- Creatively make use of `sectionColor` to make pages more visually appealing. Do not use a `sectionColor` on blocks that are immediately next to each other, unless they're intended to be coupled together. If they're intended to be coupled together, make sure to use the same color.
22-
- `topPadding` and `bottomPadding` will default to `large` if not set. Only set them if you're trying to make padding smaller to make adjacent blocks visually coupled.
23-
24-
Handling media:
25-
- If the html contains any media references, such as images, upload each of the media FIRST before exporting the page
13+
Handling images:
14+
- If the markdown contains any image, upload each of them as media FIRST before exporting the page
2615
- Do not try to upload base64 encoded media, only media that has a valid url
27-
- Only upload media that lives on the same domain as the website the html came from. Ignore all other media.
2816
- Only upload media that has a file extension
2917
- Use the `upload-media` tool to upload each media, giving the url of the media to download as a parameter
30-
- The `upload-media` tool will output the media id. Use the id as the value for any media field in the `Page` type
18+
- The `upload-media` tool will output the media ID. Use the ID as the value for any media field in the `Page` type
3119

32-
Do not explain your process or output. Do not ask clarifying questions. Do not make any assumptions. If you don't know a particular piece of information, just leave it out. Do not assume the name of the church.
20+
Handling page content:
21+
- Creatively make use of `sectionColor` to make pages more visually appealing. Do not use a `sectionColor` on blocks that are immediately next to each other, unless they're intended to be coupled together. If they're intended to be coupled together, make sure to use the same color.
22+
- `topPadding` and `bottomPadding` will default to `large` if not set. Only set them if you're trying to make padding smaller to make adjacent blocks visually coupled.
23+
- Never include a media object, instead use it's media ID.
24+
25+
Do not ask clarifying questions. Do not assume the name of the church.
3326

3427
Use the `export-page` tool only once to provide the converted data. After exporting the page, end the chat.
3528

3629
The TypeScript type definition the JSON object should abide by is the `Page` type below:
37-
<page_type>
30+
<page_types>
3831
{{.TypeScriptFile}}
39-
</page_type>
40-
41-
Below is a sample conversation from html to the `Page` json object:
42-
43-
<sample_conversion>
44-
45-
<sample_html>
46-
{{.SampleHTML}}
47-
</sample_html>
48-
49-
<assistant_tool_call_page_json>
50-
{{.SamplePageData}}
51-
</assistant_tool_call_page_json>
52-
53-
</sample_conversion>
32+
</page_types>

0 commit comments

Comments
 (0)