Only extract the first element as title
This commit is contained in:
carsten committed
1 parent
a5fec64225
commit
fec68c9645
3 files changed
+37
-2
No files matched your search
+3
-2
@@ -7,7 +7,6 @@ package readability
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"golang.org/x/text/encoding/htmlindex"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
@@ -15,6 +14,8 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/text/encoding/htmlindex"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
@@ -129,7 +130,7 @@ func readDocumentWithEncoding(content []byte) (*goquery.Document, error) {
|
||||
}
|
||||
|
||||
func getTitle(document *goquery.Document) string {
|
||||
title := document.Find("title").Text()
|
||||
title := document.Find("title").First().Text()
|
||||
return title
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package readability_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"git.kopis.de/carsten/readability"
|
||||
)
|
||||
|
||||
func TestFullHtml(t *testing.T) {
|
||||
b, err := os.ReadFile("test1.html")
|
||||
if err != nil {
|
||||
t.Fatal("Can not read file")
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(b)
|
||||
title, _, err := readability.ExtractContent(".", reader)
|
||||
if err != nil {
|
||||
t.Fatal("Can not extract content")
|
||||
}
|
||||
if title != "The Verge" {
|
||||
t.Fatalf("Unexpected title: %s", title)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
<!DOCTYPE html><html lang="en-US"><meta name="bingbot" content="nocache"/><head><meta charSet="utf-8"/><title>The Verge</title><meta name="description" content="The Verge is about technology and how it makes us feel. Founded in 2011, we offer our audience everything from breaking news to reviews to award-winning features and investigations, on our site, in video, and in podcasts."/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:site" content="@verge"/><meta property="fb:app_id" content="549923288395304"/><meta property="og:title" content="The Verge"/><meta property="og:description" content="The Verge is about technology and how it makes us feel. Founded in 2011, we offer our audience everything from breaking news to reviews to award-winning features and investigations, on our site, in video, and in podcasts."/><meta property="og:url" content="https://www.theverge.com/"/><meta property="og:type" content="website"/><meta property="og:image" content="https://platform.theverge.com/wp-content/uploads/sites/2/2024/10/the_verge_social_share.png"/><meta property="og:site_name" content="The Verge"/><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"/><meta name="apple-mobile-web-app-title" content="Verge"/><meta name="google-site-verification" content="IucFf_TKtbFFH8_YeFyEteQIwYPdANM1R46_U9DpAr4"/><link rel="alternate" type="application/rss+xml" title="The Verge" href="/rss/index.xml"/><script type="application/ld+json">[{"@context":"https://schema.org","@type":"NewsMediaOrganization","@id":"https://www.theverge.com/#publisher","name":"The Verge","url":"https://www.theverge.com","description":"The Verge is a technology and science news website founded in 2011. It offers its readers breaking tech news, tech reviews, award-winning features and investigations through articles, videos, podcasts, and newsletters.","alternateName":["Verge","theverge"],"logo":{"@type":"ImageObject","url":"https://cdn.vox-cdn.com/uploads/chorus_asset/file/24018771/verge_duet_500.png","height":"120","width":"500"},"ethicsPolicy":"https://www.theverge.com/ethics-statement","masthead":"https://www.theverge.com/about-the-verge","foundingDate":"2011-11-01","founder":[{"@type":"Person","name":"Jim Bankoff"},{"@type":"Person","name":"Joshua Topolsky"},{"@type":"Person","name":"Marty Moe"},{"@type":"Person","name":"Nilay Patel"},{"@type":"Person","name":"Joanna Stern"},{"@type":"Person","name":"Dieter Bohn"},{"@type":"Person","name":"Joseph L. Flatley"},{"@type":"Person","name":"Sean Hollister"},{"@type":"Person","name":"Thomas Houston"},{"@type":"Person","name":"Laura June"},{"@type":"Person","name":"Paul Miller"},{"@type":"Person","name":"Ross Miller"},{"@type":"Person","name":"Thomas Ricker"},{"@type":"Person","name":"Vald Savov"},{"@type":"Person","name":"Chris Ziegler"}],"ownershipFundingInfo":"The Verge is owned by Vox Media, the leading modern media company dedicated to high-quality journalism.","sameAs":["https://www.facebook.com/verge/","https://twitter.com/verge","https://www.instagram.com/verge","https://www.youtube.com/user/theverge","https://www.linkedin.com/company/the-verge","https://www.wikidata.org/wiki/Q2155442","https://en.wikipedia.org/wiki/The_Verge"],"knowsAbout":[{"@type":"Thing","name":"technology"},{"@type":"Thing","name":"tech news"},{"@type":"Thing","name":"tech products"},{"@type":"Thing","name":"tech events"},{"@type":"Thing","name":"consumer electronics"},{"@type":"Thing","name":"gadgets"},{"@type":"Thing","name":"software news"},{"@type":"Thing","name":"tech reviews"},{"@type":"Thing","name":"the verge reviews"},{"@type":"Thing","name":"phone reviews"},{"@type":"Thing","name":"laptop reviews"},{"@type":"Thing","name":"buying guides"},{"@type":"Thing","name":"science news"},{"@type":"Thing","name":"cybersecurity news"},{"@type":"Thing","name":"gaming news"},{"@type":"Thing","name":"digital creator news"},{"@type":"Thing","name":"social media platforms"},{"@type":"Thing","name":"streaming platforms"},{"@type":"Thing","name":"section 230"}],"award":["The National Magazine Awards 2022 Winner in Best Digital Illustration","The Pulitzer Prize 2020 Finalist in Feature Writing","Gerald Loeb 2022 Award Winner in Feature","Deadline Club Award 2021 Winner in Reporting by Independent Digital Media"]},{"@context":"https://schema.org","@type":"WebSite","url":"https://www.theverge.com/","name":"The Verge","potentialAction":{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https://www.theverge.com/search?q={search_term_string}"},"query-input":"required name=search_term_string"}}]</script><meta name="robots" content="index,follow"/><link rel="canonical" href="https://www.theverge.com"/><link rel="preload" as="image" imageSrcSet="https://platform.theverge.com/wp-content/uploads/sites/2/2025/06/NintendoSwitch2_MarioKartWorld_Character_SCRN_25.png?quality=90&strip=all&crop=7.8125%2C0%2C84.375%2C100&w=256 256w, https://platform.theverge.com/wpLine truncated
|
||||
:root {
|
||||
--font-fkroman: '__fkRomanStandard_cfceed', '__fkRomanStandard_Fallback_cfceed', Georgia, serif;--font-manuka: '__manuka_62afb5', '__manuka_Fallback_62afb5', Impact, Helvetica, sans-serif;--font-polysans: '__polySans_9afc27', '__polySans_Fallback_9afc27', Helvetica, Arial, sans-serif;--font-polysans-mono: '__polySansMono_0b836e', '__polySansMono_Fallback_0b836e', Courier New, Courier, monospace;
|
||||
}
|
||||
</style><style>
|
||||
*, *::before, *::after {
|
||||
transition: none!important;
|
||||
}
|
||||
</style><div class="duet--app _1ymtmqp1"><a class="duet--cta--button _1f7jm892 _1f7jm890 duet--cta--skip-to-content _8mk3zk0" href="#content"><span>Skip to main content</span></a><div class="_10nsm93q _10nsm937 _10nsm930 _10nsm932c"><div class="_10nsm930 _10nsm93r" data-concert="leaderboard_top_tablet_desktop"></div></div><div class="bi882e7"><div class="bi882e0"><div class="_1ckl8uw3 _1ckl8uw0"><div class="_1ckl8uw4"><div class="_1wnt6o3z"></div><nav class="_1ckl8uwc _1ckl8uwb _1wnt6o37 _1wnt6o31" aria-label="Top Navigation"><ul class="_18l06ho1 _18l06ho0"><a href="/" class="_1wnt6o3w"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 309 70" role="img" width="100%" height="100%" fill="currentColor"><title>The Verge</title><desc>The Verge logo.</desc><path d="m231.196 17.897-.302 9.071c-10.592-.726-13.618 1.996-13.618 10.885V39h-9.078V18.441h9.078v5.866c2.724-4.777 6.416-6.954 13.92-6.41ZM15.131 54.786h9.078V19.71h-9.078v35.075Zm44.968-36.828c-6.355 0-10.228 2.842-12.286 5.986V4.593H0v8.466h39.34V39h8.654c0-7.438 4.298-12.697 9.563-12.697 4.54 0 6.597 2.237 6.597 10.28v18.203h9.078V33.318c0-10.28-5.265-15.36-13.133-15.36ZM95.807 47.83c-5.507 0-9.078-3.326-9.683-8.829H77.59c.847 9.676 7.202 16.51 18.157 16.51 8.473 0 13.254-3.81 15.736-9.555l-7.687-3.387c-1.15 3.447-3.268 5.261-7.989 5.261Zm-.363-29.692a19.226 19.226 0 0 0-9.32 2.177l4.357 6.168c1.634-.846 3.39-1.27 5.266-1.21 5.084 0 7.686 3.327 8.049 7.68H95.02v6.048h17.31c.121-.907.182-1.754.182-2.66.06-13.184-8.655-18.203-17.068-18.203ZM185.32 47.83c-5.507 0-9.078-3.326-9.683-8.829h-8.534c.847 9.676 7.202 16.51 18.157 16.51 8.473 0 13.254-3.81 15.736-9.555l-7.687-3.387c-1.21 3.447-3.328 5.261-7.989 5.261Zm-.302-29.692a19.226 19.226 0 0 0-9.321 2.177l4.358 6.168c1.634-.846 3.389-1.27 5.265-1.21 5.084 0 7.687 3.327 8.05 7.68h-8.776v6.048h17.31c.121-.907.181-1.754.181-2.66.061-13.184-8.655-18.203-17.067-18.203ZM291.416 47.83c-5.507 0-9.078-3.326-9.683-8.829h-8.534c.847 9.676 7.202 16.51 18.157 16.51 8.473 0 13.254-3.81 15.736-9.555l-7.687-3.387c-1.21 3.447-3.328 5.261-7.989 5.261Zm-.484-29.692a19.225 19.225 0 0 0-9.32 2.177l4.357 6.168c1.635-.846 3.39-1.27 5.266-1.21 5.084 0 7.686 3.327 8.049 7.68h-8.775v6.048h17.309c.121-.907.182-1.754.182-2.66.06-13.184-8.655-18.203-17.068-18.203ZM117.172.299 133.5 39h9.926L130.971 8.221h16.099V.36L117.172.3Zm48.418.06L146.888 47.71l-2.784 7.076h9.502L176.06.36h-10.47Zm83.461 53.58c3.873 0 7.081-1.089 9.32-2.963l-3.631-5.745c-1.15.484-2.421.665-3.692.665-4.963 0-7.808-2.963-8.776-6.894h-8.897c1.211 8.406 7.263 14.937 15.676 14.937Zm11.196-30.418c-2.057-3.265-6.234-5.624-12.044-5.624-5.689-.06-10.954 3.024-13.738 8.043l7.565 4.838c1.392-2.903 4.116-4.838 8.292-4.838 5.931 0 9.925 4.596 9.925 10.038 0 1.029-.121 2.057-.423 3.024h9.502v-20.5h-9.079v5.019Zm-8.775 38.642c-5.871 0-8.05-2.842-8.474-6.168h-8.654c.181 6.35 4.418 13.304 17.309 13.304 8.715 0 14.404-4.354 16.765-10.885l-8.171-2.842c-1.15 4.233-4.297 6.591-8.775 6.591Z"></path></svg></a><li class="_1wnt6o3v _1wnt6o3r _1wnt6o3l urbcaf0"><a href="/tech" class="urbcaf4"><span class="urbcaf5 urbcaf8 urbcaf7">Tech</span></a></li><li class="_1wnt6o3v _1wnt6o3r _1wnt6o3l urbcaf0"><a href="/reviews" class="urbcaf4"><span class="urbcaf5 urbcaf8 urbcaf7">Reviews</span></a></li><li class="_1wnt6o3v _1wnt6o3r _1wnt6o3l urbcaf0"><a href="/science" class="urbcaf4"><span class="urbcaf5 urbcaf8 urbcaf7">Science</span></a></li><li class="_1wnt6o3v _1wnt6o3r _1wnt6o3l urbcaf0"><a href="/entertainment" class="urbcaf4"><span class="urbcaf5 urbcaf8 urbcaf7">Entertainment</span></a></li><li class="_1wnt6o3v _1wnt6o3r _1wnt6o3l urbcaf0"><a href="/ai-artificial-intelligence" class="urbcaf4"><span class="urbcaf5 urbcaf8 urbcaf7">AI</span></a></li><li class="_1wnt6o3v _1wnt6o3r _1wnt6o3l _1wnt6o3y urbcaf0"><button class="urbcaf4"><span class="urbcaf5 urbcaf8 urbcaf7">More<svg class="_1wnt6o30" width="22" height="22" viewBox="0 0 29 30" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><title>Expand</title><line x1="4" y1="14.4805" x2="26" y2="14.4805" stroke="currentColor" stroke-width="4"></line><line x1="15" y1="3.48047" x2="15" y2="25.4805" stroke="currentColor" stroke-width="4"></line></svg></span></button></li></ul></nav></div></div></div><div class="bi882e1"><div class="_1bjoufy1 _1wnt6o3r"><div class="vbsnq72"><div class="vbsnq75 _1bjoufy3"><div class="_1bjoufy2"><button class="_16902kx0 _1wnt6o3r" aria-controls="drawer_:R196:" aria-label="Open Drawer"><div class="_16902kx3"><span class="_16902kx1">Menu</span><span class="_16902kx2"><svg class="_16902kxd" width="29" height="30" viewBox="0 0 29 30" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><title>Expand</title><line x1="4" y1="14.4805" x2="26" y2="14.4805" stroke="currentColor" stroke-width="4"></line><line x1="15" y1="3.48047" x2="15" y2="25.4805" stroke="currentColor" stroke-width="4"></line></svg></span></div></button></div></div></div></div></div></div><div class="_1huz5gz0" id="drawer_:R196:" role="dialLine truncated
|
||||
Reference in new issue
Block a user