348 lines
9.3 KiB
Go
348 lines
9.3 KiB
Go
package dataset
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const maxFeedBytes = 5 << 20
|
|
|
|
var (
|
|
ErrFeedFormat = errors.New("unsupported feed format")
|
|
ErrFeedTooLarge = errors.New("feed response is too large")
|
|
ErrUnsafeFeedURL = errors.New("feed url resolves to a private or local address")
|
|
htmlTagPattern = regexp.MustCompile(`<[^>]*>`)
|
|
)
|
|
|
|
type FeedItem struct {
|
|
ExternalID string
|
|
Title string
|
|
Summary string
|
|
Content string
|
|
URL string
|
|
PublishedAt *time.Time
|
|
}
|
|
|
|
type ParsedFeed struct {
|
|
Format string
|
|
Items []FeedItem
|
|
}
|
|
|
|
type FeedFetcher interface {
|
|
Fetch(context.Context, string) (ParsedFeed, error)
|
|
}
|
|
|
|
type HTTPFeedFetcher struct {
|
|
client *http.Client
|
|
}
|
|
|
|
func NewHTTPFeedFetcher() *HTTPFeedFetcher {
|
|
transport := &http.Transport{
|
|
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
|
host, port, err := net.SplitHostPort(address)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
dialer := &net.Dialer{Timeout: 10 * time.Second}
|
|
var lastErr error
|
|
for _, candidate := range ips {
|
|
if !isPublicIP(candidate.IP) {
|
|
continue
|
|
}
|
|
connection, err := dialer.DialContext(ctx, network, net.JoinHostPort(candidate.IP.String(), port))
|
|
if err == nil {
|
|
return connection, nil
|
|
}
|
|
lastErr = err
|
|
}
|
|
if lastErr != nil {
|
|
return nil, lastErr
|
|
}
|
|
return nil, ErrUnsafeFeedURL
|
|
},
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
ResponseHeaderTimeout: 15 * time.Second,
|
|
}
|
|
fetcher := &HTTPFeedFetcher{}
|
|
fetcher.client = &http.Client{
|
|
Transport: transport,
|
|
Timeout: 30 * time.Second,
|
|
CheckRedirect: func(request *http.Request, via []*http.Request) error {
|
|
if len(via) >= 5 {
|
|
return errors.New("too many feed redirects")
|
|
}
|
|
return validateFeedURL(request.URL)
|
|
},
|
|
}
|
|
return fetcher
|
|
}
|
|
|
|
func (f *HTTPFeedFetcher) Fetch(ctx context.Context, value string) (ParsedFeed, error) {
|
|
parsedURL, err := url.Parse(value)
|
|
if err != nil {
|
|
return ParsedFeed{}, err
|
|
}
|
|
if err := validateFeedURL(parsedURL); err != nil {
|
|
return ParsedFeed{}, err
|
|
}
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsedURL.String(), nil)
|
|
if err != nil {
|
|
return ParsedFeed{}, err
|
|
}
|
|
request.Header.Set("Accept", "application/feed+json, application/json, application/rss+xml, application/atom+xml, application/xml, text/xml;q=0.9")
|
|
request.Header.Set("User-Agent", "SenlinAI-Agent/1.0")
|
|
response, err := f.client.Do(request)
|
|
if err != nil {
|
|
return ParsedFeed{}, err
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
|
|
return ParsedFeed{}, fmt.Errorf("feed request returned %s", response.Status)
|
|
}
|
|
body, err := io.ReadAll(io.LimitReader(response.Body, maxFeedBytes+1))
|
|
if err != nil {
|
|
return ParsedFeed{}, err
|
|
}
|
|
if len(body) > maxFeedBytes {
|
|
return ParsedFeed{}, ErrFeedTooLarge
|
|
}
|
|
return ParseFeed(body, response.Header.Get("Content-Type"))
|
|
}
|
|
|
|
func ParseFeed(body []byte, contentType string) (ParsedFeed, error) {
|
|
body = bytes.TrimSpace(bytes.TrimPrefix(body, []byte{0xef, 0xbb, 0xbf}))
|
|
if len(body) == 0 {
|
|
return ParsedFeed{}, ErrFeedFormat
|
|
}
|
|
lowerContentType := strings.ToLower(contentType)
|
|
if body[0] == '{' || strings.Contains(lowerContentType, "json") {
|
|
if feed, err := parseJSONFeed(body); err == nil {
|
|
return feed, nil
|
|
}
|
|
}
|
|
if body[0] == '<' || strings.Contains(lowerContentType, "xml") || strings.Contains(lowerContentType, "rss") || strings.Contains(lowerContentType, "atom") {
|
|
return parseXMLFeed(body)
|
|
}
|
|
return ParsedFeed{}, ErrFeedFormat
|
|
}
|
|
|
|
type jsonFeedDocument struct {
|
|
Version string `json:"version"`
|
|
Items []jsonFeedItem `json:"items"`
|
|
}
|
|
|
|
type jsonFeedItem struct {
|
|
ID string `json:"id"`
|
|
URL string `json:"url"`
|
|
Title string `json:"title"`
|
|
Summary string `json:"summary"`
|
|
ContentText string `json:"content_text"`
|
|
ContentHTML string `json:"content_html"`
|
|
DatePublished string `json:"date_published"`
|
|
}
|
|
|
|
func parseJSONFeed(body []byte) (ParsedFeed, error) {
|
|
var document jsonFeedDocument
|
|
if err := json.Unmarshal(body, &document); err != nil {
|
|
return ParsedFeed{}, err
|
|
}
|
|
if document.Version == "" {
|
|
return ParsedFeed{}, ErrFeedFormat
|
|
}
|
|
items := make([]FeedItem, 0, len(document.Items))
|
|
for _, input := range document.Items {
|
|
content := strings.TrimSpace(input.ContentText)
|
|
if content == "" {
|
|
content = plainText(input.ContentHTML)
|
|
}
|
|
item := normalizeFeedItem(FeedItem{
|
|
ExternalID: input.ID, Title: plainText(input.Title), Summary: plainText(input.Summary),
|
|
Content: content, URL: input.URL, PublishedAt: parseFeedTime(input.DatePublished),
|
|
})
|
|
if item.Title != "" {
|
|
items = append(items, item)
|
|
}
|
|
}
|
|
return ParsedFeed{Format: "json_feed", Items: items}, nil
|
|
}
|
|
|
|
type xmlFeedDocument struct {
|
|
XMLName xml.Name
|
|
Channel struct {
|
|
Items []xmlFeedItem `xml:"item"`
|
|
} `xml:"channel"`
|
|
Items []xmlFeedItem `xml:"item"`
|
|
Entries []xmlAtomEntry `xml:"entry"`
|
|
}
|
|
|
|
type xmlFeedItem struct {
|
|
Title string `xml:"title"`
|
|
Link string `xml:"link"`
|
|
GUID string `xml:"guid"`
|
|
Description string `xml:"description"`
|
|
Content string `xml:"encoded"`
|
|
PubDate string `xml:"pubDate"`
|
|
Date string `xml:"date"`
|
|
}
|
|
|
|
type xmlAtomEntry struct {
|
|
Title string `xml:"title"`
|
|
ID string `xml:"id"`
|
|
Links []xmlAtomLink `xml:"link"`
|
|
Summary string `xml:"summary"`
|
|
Content string `xml:"content"`
|
|
Published string `xml:"published"`
|
|
Updated string `xml:"updated"`
|
|
}
|
|
|
|
type xmlAtomLink struct {
|
|
Href string `xml:"href,attr"`
|
|
Rel string `xml:"rel,attr"`
|
|
}
|
|
|
|
func parseXMLFeed(body []byte) (ParsedFeed, error) {
|
|
var document xmlFeedDocument
|
|
if err := xml.Unmarshal(body, &document); err != nil {
|
|
return ParsedFeed{}, err
|
|
}
|
|
switch strings.ToLower(document.XMLName.Local) {
|
|
case "rss", "rdf":
|
|
inputs := document.Channel.Items
|
|
if len(inputs) == 0 {
|
|
inputs = document.Items
|
|
}
|
|
items := make([]FeedItem, 0, len(inputs))
|
|
for _, input := range inputs {
|
|
content := plainText(input.Content)
|
|
if content == "" {
|
|
content = plainText(input.Description)
|
|
}
|
|
item := normalizeFeedItem(FeedItem{
|
|
ExternalID: input.GUID, Title: plainText(input.Title),
|
|
Summary: plainText(input.Description), Content: content,
|
|
URL: strings.TrimSpace(input.Link), PublishedAt: parseFeedTime(firstNonEmpty(input.PubDate, input.Date)),
|
|
})
|
|
if item.Title != "" {
|
|
items = append(items, item)
|
|
}
|
|
}
|
|
return ParsedFeed{Format: "rss", Items: items}, nil
|
|
case "feed":
|
|
items := make([]FeedItem, 0, len(document.Entries))
|
|
for _, input := range document.Entries {
|
|
item := normalizeFeedItem(FeedItem{
|
|
ExternalID: input.ID, Title: plainText(input.Title),
|
|
Summary: plainText(input.Summary), Content: plainText(input.Content),
|
|
URL: atomLink(input.Links), PublishedAt: parseFeedTime(firstNonEmpty(input.Published, input.Updated)),
|
|
})
|
|
if item.Title != "" {
|
|
items = append(items, item)
|
|
}
|
|
}
|
|
return ParsedFeed{Format: "atom", Items: items}, nil
|
|
default:
|
|
return ParsedFeed{}, ErrFeedFormat
|
|
}
|
|
}
|
|
|
|
func normalizeFeedItem(item FeedItem) FeedItem {
|
|
item.Title = truncateRunes(strings.TrimSpace(item.Title), 500)
|
|
item.Summary = strings.TrimSpace(item.Summary)
|
|
item.Content = strings.TrimSpace(item.Content)
|
|
item.URL = truncateRunes(strings.TrimSpace(item.URL), 2048)
|
|
key := firstNonEmpty(strings.TrimSpace(item.ExternalID), item.URL)
|
|
if key == "" {
|
|
published := ""
|
|
if item.PublishedAt != nil {
|
|
published = item.PublishedAt.UTC().Format(time.RFC3339Nano)
|
|
}
|
|
key = item.Title + "\x00" + published
|
|
}
|
|
sum := sha256.Sum256([]byte(key))
|
|
item.ExternalID = hex.EncodeToString(sum[:])
|
|
return item
|
|
}
|
|
|
|
func parseFeedTime(value string) *time.Time {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return nil
|
|
}
|
|
layouts := []string{
|
|
time.RFC3339Nano, time.RFC3339, time.RFC1123Z, time.RFC1123,
|
|
time.RFC822Z, time.RFC822, time.RFC850, time.ANSIC,
|
|
"Mon, 2 Jan 2006 15:04:05 -0700",
|
|
}
|
|
for _, layout := range layouts {
|
|
if parsed, err := time.Parse(layout, value); err == nil {
|
|
utc := parsed.UTC()
|
|
return &utc
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func plainText(value string) string {
|
|
value = htmlTagPattern.ReplaceAllString(value, " ")
|
|
return strings.Join(strings.Fields(html.UnescapeString(value)), " ")
|
|
}
|
|
|
|
func atomLink(links []xmlAtomLink) string {
|
|
for _, link := range links {
|
|
if link.Rel == "" || link.Rel == "alternate" {
|
|
return strings.TrimSpace(link.Href)
|
|
}
|
|
}
|
|
if len(links) > 0 {
|
|
return strings.TrimSpace(links[0].Href)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func firstNonEmpty(values ...string) string {
|
|
for _, value := range values {
|
|
if strings.TrimSpace(value) != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func validateFeedURL(value *url.URL) error {
|
|
if value == nil || (value.Scheme != "http" && value.Scheme != "https") || value.Hostname() == "" || value.User != nil {
|
|
return ErrURLInvalid
|
|
}
|
|
if ip := net.ParseIP(value.Hostname()); ip != nil && !isPublicIP(ip) {
|
|
return ErrUnsafeFeedURL
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func isPublicIP(ip net.IP) bool {
|
|
return ip != nil &&
|
|
!ip.IsLoopback() &&
|
|
!ip.IsPrivate() &&
|
|
!ip.IsLinkLocalUnicast() &&
|
|
!ip.IsLinkLocalMulticast() &&
|
|
!ip.IsUnspecified() &&
|
|
!ip.IsMulticast()
|
|
}
|