1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
package rag
import (
"archive/zip"
"bytes"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/ledongthuc/pdf"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/extension"
"github.com/yuin/goldmark/parser"
"github.com/yuin/goldmark/renderer/html"
)
func ExtractText(fpath string) (string, error) {
ext := strings.ToLower(path.Ext(fpath))
switch ext {
case ".txt":
return extractTextFromFile(fpath)
case ".md", ".markdown":
return extractTextFromMarkdown(fpath)
case ".html", ".htm":
return extractTextFromHtmlFile(fpath)
case ".epub":
return extractTextFromEpub(fpath)
case ".pdf":
return extractTextFromPdf(fpath)
default:
return "", fmt.Errorf("unsupported file format: %s", ext)
}
}
func extractTextFromFile(fpath string) (string, error) {
data, err := os.ReadFile(fpath)
if err != nil {
return "", err
}
return string(data), nil
}
func extractTextFromHtmlFile(fpath string) (string, error) {
data, err := os.ReadFile(fpath)
if err != nil {
return "", err
}
return extractTextFromHtmlContent(data)
}
// non utf-8 encoding?
func extractTextFromHtmlContent(data []byte) (string, error) {
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(data))
if err != nil {
return "", err
}
// Remove script and style tags
doc.Find("script, style, noscript").Each(func(i int, s *goquery.Selection) {
s.Remove()
})
// Get text and clean it
text := doc.Text()
// Collapse all whitespace (newlines, tabs, multiple spaces) into single spaces
cleaned := strings.Join(strings.Fields(text), " ")
return cleaned, nil
}
func extractTextFromMarkdown(fpath string) (string, error) {
data, err := os.ReadFile(fpath)
if err != nil {
return "", err
}
// Convert markdown to HTML
md := goldmark.New(
goldmark.WithExtensions(extension.GFM),
goldmark.WithParserOptions(parser.WithAutoHeadingID()),
goldmark.WithRendererOptions(html.WithUnsafe()), // allow raw HTML if needed
)
var buf bytes.Buffer
if err := md.Convert(data, &buf); err != nil {
return "", err
}
// Now extract text from the resulting HTML (using goquery or similar)
return extractTextFromHtmlContent(buf.Bytes())
}
func extractTextFromEpub(fpath string) (string, error) {
r, err := zip.OpenReader(fpath)
if err != nil {
return "", fmt.Errorf("failed to open epub: %w", err)
}
defer r.Close()
var sb strings.Builder
for _, f := range r.File {
ext := strings.ToLower(path.Ext(f.Name))
if ext != ".xhtml" && ext != ".html" && ext != ".htm" && ext != ".xml" {
continue
}
// Skip manifest, toc, ncx files - they don't contain book content
nameLower := strings.ToLower(f.Name)
if strings.Contains(nameLower, "toc") || strings.Contains(nameLower, "nav") ||
strings.Contains(nameLower, "manifest") || strings.Contains(nameLower, ".opf") ||
strings.HasSuffix(nameLower, ".ncx") {
continue
}
rc, err := f.Open()
if err != nil {
continue
}
if sb.Len() > 0 {
sb.WriteString("\n\n")
}
sb.WriteString(f.Name)
sb.WriteString("\n")
buf, readErr := io.ReadAll(rc)
rc.Close()
if readErr == nil {
sb.WriteString(stripHTML(string(buf)))
}
}
if sb.Len() == 0 {
return "", errors.New("no content extracted from epub")
}
return sb.String(), nil
}
func stripHTML(html string) string {
var sb strings.Builder
inTag := false
for _, r := range html {
switch r {
case '<':
inTag = true
case '>':
inTag = false
default:
if !inTag {
sb.WriteRune(r)
}
}
}
return sb.String()
}
func extractTextFromPdf(fpath string) (string, error) {
_, err := exec.LookPath("pdftotext")
if err == nil {
out, err := exec.Command("pdftotext", "-layout", fpath, "-").Output()
if err == nil && len(out) > 0 {
return string(out), nil
}
}
return extractTextFromPdfPureGo(fpath)
}
func extractTextFromPdfPureGo(fpath string) (string, error) {
df, r, err := pdf.Open(fpath)
if err != nil {
return "", fmt.Errorf("failed to open pdf: %w", err)
}
defer df.Close()
textReader, err := r.GetPlainText()
if err != nil {
return "", fmt.Errorf("failed to extract text from pdf: %w", err)
}
var buf bytes.Buffer
_, err = io.Copy(&buf, textReader)
if err != nil {
return "", fmt.Errorf("failed to read pdf text: %w", err)
}
return buf.String(), nil
}
|
