all repos

rss-tools @ 19e8c1c7e354df8337b753281d599f9c45a33036

get rss feed from sources that(i need and) dont provide one
4 files changed, 171 insertions(+), 20 deletions(-)
telegram: preserve \n in multiline messages, support multi image messages
Author: Oleksandr Smirnov olexsmir@gmail.com
Committed at: 2026-05-23 15:55:44 +0300
Authored at: 2026-05-23 15:01:24 +0300
Change ID: okznmtmtvokuxnlyltzzntmsvmvwkmxr
Parent: 11fd04d
M sources/telegram/sdk.go
···
        52
        52
         }

      
        53
        53
         

      
        54
        54
         type Message struct {

      
        55
        
        -	MessageID     int64             `json:"message_id"`

      
        56
        
        -	From          *User             `json:"from"`

      
        57
        
        -	Chat          *Chat             `json:"chat"`

      
        58
        
        -	Text          string            `json:"text"`

      
        59
        
        -	Caption       string            `json:"caption,omitempty"`

      
        60
        
        -	Date          int64             `json:"date"`

      
        61
        
        -	Photo         []PhotoSize       `json:"photo,omitempty"`

      
        62
        
        -	PhotoBase64   string            `json:"photo_base64,omitempty"`

      
        63
        
        -	PhotoMIMEType string            `json:"photo_mime_type,omitempty"`

      
        64
        
        -	LinkTitles    map[string]string `json:"-"`

      
        
        55
        +	MessageID        int64             `json:"message_id"`

      
        
        56
        +	From             *User             `json:"from"`

      
        
        57
        +	Chat             *Chat             `json:"chat"`

      
        
        58
        +	Text             string            `json:"text"`

      
        
        59
        +	Caption          string            `json:"caption,omitempty"`

      
        
        60
        +	Date             int64             `json:"date"`

      
        
        61
        +	MediaGroupID     string            `json:"media_group_id,omitempty"`

      
        
        62
        +	Photo            []PhotoSize       `json:"photo,omitempty"`

      
        
        63
        +	PhotoBase64      string            `json:"photo_base64,omitempty"`

      
        
        64
        +	PhotoMIMEType    string            `json:"photo_mime_type,omitempty"`

      
        
        65
        +	PhotoAttachments []PhotoAttachment `json:"photo_attachments,omitempty"`

      
        
        66
        +	LinkTitles       map[string]string `json:"-"`

      
        
        67
        +}

      
        
        68
        +

      
        
        69
        +type PhotoAttachment struct {

      
        
        70
        +	Base64   string `json:"base64,omitempty"`

      
        
        71
        +	MIMEType string `json:"mime_type,omitempty"`

      
        65
        72
         }

      
        66
        73
         

      
        67
        74
         type PhotoSize struct {

      ···
        90
        97
         		if err != nil {

      
        91
        98
         			return nil, err

      
        92
        99
         		}

      
        93
        
        -		msg.PhotoBase64 = base64.StdEncoding.EncodeToString(data)

      
        94
        
        -		msg.PhotoMIMEType = mimeType

      
        
        100
        +		attachment := PhotoAttachment{

      
        
        101
        +			Base64:   base64.StdEncoding.EncodeToString(data),

      
        
        102
        +			MIMEType: mimeType,

      
        
        103
        +		}

      
        
        104
        +		msg.PhotoAttachments = append(msg.PhotoAttachments, attachment)

      
        
        105
        +		if msg.PhotoBase64 == "" {

      
        
        106
        +			msg.PhotoBase64 = attachment.Base64

      
        
        107
        +			msg.PhotoMIMEType = attachment.MIMEType

      
        
        108
        +		}

      
        95
        109
         	}

      
        96
        110
         	return resp.Result, nil

      
        97
        111
         }

      
M sources/telegram/sdk_test.go
···
        59
        59
         	msg := updates[0].Message

      
        60
        60
         	is.Equal(t, "large", seenGetFileForID)

      
        61
        61
         	is.Equal(t, "photo msg", msg.Caption)

      
        
        62
        +	is.Equal(t, 1, len(msg.PhotoAttachments))

      
        
        63
        +	is.Equal(t, base64.StdEncoding.EncodeToString(pngData), msg.PhotoAttachments[0].Base64)

      
        
        64
        +	is.Equal(t, "image/png", msg.PhotoAttachments[0].MIMEType)

      
        62
        65
         	is.Equal(t, base64.StdEncoding.EncodeToString(pngData), msg.PhotoBase64)

      
        63
        66
         	is.Equal(t, "image/png", msg.PhotoMIMEType)

      
        64
        67
         }

      
M sources/telegram/telegram.go
···
        6
        6
         	"encoding/binary"

      
        7
        7
         	"encoding/gob"

      
        8
        8
         	"fmt"

      
        
        9
        +	"html"

      
        9
        10
         	"log/slog"

      
        10
        11
         	"net/http"

      
        11
        12
         	"strings"

      ···
        69
        70
         				return

      
        70
        71
         			}

      
        71
        72
         		}

      
        
        73
        +	}

      
        
        74
        +

      
        
        75
        +	for _, m := range groupMessages(messages) {

      
        72
        76
         		feed.Add(feedEntryFromMessage(m))

      
        73
        77
         	}

      
        74
        78
         

      ···
        166
        170
         		return nil

      
        167
        171
         	})

      
        168
        172
         	return messages, err

      
        
        173
        +}

      
        
        174
        +

      
        
        175
        +func groupMessages(messages []*Message) []*Message {

      
        
        176
        +	if len(messages) == 0 {

      
        
        177
        +		return messages

      
        
        178
        +	}

      
        
        179
        +

      
        
        180
        +	groups := make(map[string]*Message)

      
        
        181
        +	out := make([]*Message, 0, len(messages))

      
        
        182
        +	for _, m := range messages {

      
        
        183
        +		if m == nil || strings.TrimSpace(m.MediaGroupID) == "" {

      
        
        184
        +			out = append(out, m)

      
        
        185
        +			continue

      
        
        186
        +		}

      
        
        187
        +

      
        
        188
        +		group, ok := groups[m.MediaGroupID]

      
        
        189
        +		if !ok {

      
        
        190
        +			group = &Message{

      
        
        191
        +				MessageID:    m.MessageID,

      
        
        192
        +				From:         m.From,

      
        
        193
        +				Chat:         m.Chat,

      
        
        194
        +				Text:         m.Text,

      
        
        195
        +				Caption:      m.Caption,

      
        
        196
        +				Date:         m.Date,

      
        
        197
        +				MediaGroupID: m.MediaGroupID,

      
        
        198
        +				LinkTitles:   m.LinkTitles,

      
        
        199
        +			}

      
        
        200
        +			groups[m.MediaGroupID] = group

      
        
        201
        +			out = append(out, group)

      
        
        202
        +		}

      
        
        203
        +

      
        
        204
        +		if m.MessageID != 0 && (group.MessageID == 0 || m.MessageID < group.MessageID) {

      
        
        205
        +			group.MessageID = m.MessageID

      
        
        206
        +		}

      
        
        207
        +		if m.Date != 0 && (group.Date == 0 || m.Date < group.Date) {

      
        
        208
        +			group.Date = m.Date

      
        
        209
        +		}

      
        
        210
        +		if strings.TrimSpace(messageText(group)) == "" && strings.TrimSpace(messageText(m)) != "" {

      
        
        211
        +			group.Caption = m.Caption

      
        
        212
        +			group.Text = m.Text

      
        
        213
        +			if len(m.LinkTitles) > 0 {

      
        
        214
        +				group.LinkTitles = m.LinkTitles

      
        
        215
        +			}

      
        
        216
        +		} else if len(group.LinkTitles) == 0 && len(m.LinkTitles) > 0 {

      
        
        217
        +			group.LinkTitles = m.LinkTitles

      
        
        218
        +		}

      
        
        219
        +

      
        
        220
        +		group.PhotoAttachments = append(group.PhotoAttachments, messagePhotos(m)...)

      
        
        221
        +		if group.PhotoBase64 == "" && m.PhotoBase64 != "" {

      
        
        222
        +			group.PhotoBase64 = m.PhotoBase64

      
        
        223
        +			group.PhotoMIMEType = m.PhotoMIMEType

      
        
        224
        +		}

      
        
        225
        +	}

      
        
        226
        +	return out

      
        169
        227
         }

      
        170
        228
         

      
        171
        229
         func (t *telegram) enrichMessageWithLinkTitles(ctx context.Context, m *Message) bool {

      ···
        205
        263
         

      
        206
        264
         func feedEntryFromMessage(m *Message) app.FeedEntry {

      
        207
        265
         	updated := time.Unix(m.Date, 0)

      
        208
        
        -	text := messageText(m)

      
        
        266
        +	text := normalizeMessageText(messageText(m))

      
        209
        267
         	normalizedLinks := normalizeLinks(messageLinks(text))

      
        210
        268
         	entryID := fmt.Sprintf("telegram-%d", m.MessageID)

      
        211
        269
         	if videoID, ok := firstYouTubeVideoID(normalizedLinks); ok {

      
        212
        270
         		entryID = "yt:video:" + videoID

      
        213
        271
         	}

      
        214
        272
         

      
        215
        
        -	if m.PhotoBase64 == "" {

      
        
        273
        +	photos := messagePhotos(m)

      
        
        274
        +	if len(photos) == 0 {

      
        216
        275
         		title := text

      
        217
        276
         		if isSingleLinkMessage(text) {

      
        218
        277
         			for _, link := range normalizedLinks {

      ···
        230
        289
         		contentType := ""

      
        231
        290
         		if len(normalizedLinks) > 0 {

      
        232
        291
         			content, _ = linkifyMessageText(text)

      
        
        292
        +			content = preserveLineBreaks(content)

      
        
        293
        +			contentType = "html"

      
        
        294
        +		} else if strings.Contains(text, "\n") {

      
        
        295
        +			content = preserveLineBreaks(html.EscapeString(text))

      
        233
        296
         			contentType = "html"

      
        234
        297
         		}

      
        235
        298
         

      ···
        243
        306
         		}

      
        244
        307
         	}

      
        245
        308
         

      
        246
        
        -	parts := make([]string, 0, 2)

      
        
        309
        +	parts := make([]string, 0, 1+len(photos))

      
        247
        310
         	if t := strings.TrimSpace(text); t != "" {

      
        248
        
        -		linkified, _ := linkifyMessageText(t)

      
        
        311
        +		linkified, _ := linkifyMessageText(text)

      
        
        312
        +		linkified = preserveLineBreaks(linkified)

      
        249
        313
         		parts = append(parts, "<p>"+linkified+"</p>")

      
        250
        314
         	}

      
        251
        
        -	mimeType := m.PhotoMIMEType

      
        252
        
        -	if mimeType == "" {

      
        253
        
        -		mimeType = "image/jpeg"

      
        
        315
        +	for _, photo := range photos {

      
        
        316
        +		if photo.Base64 == "" {

      
        
        317
        +			continue

      
        
        318
        +		}

      
        
        319
        +		mimeType := photo.MIMEType

      
        
        320
        +		if mimeType == "" {

      
        
        321
        +			mimeType = "image/jpeg"

      
        
        322
        +		}

      
        
        323
        +		parts = append(parts, fmt.Sprintf(`<p><img src="data:%s;base64,%s" alt="telegram image"/></p>`, mimeType, photo.Base64))

      
        254
        324
         	}

      
        255
        
        -	parts = append(parts, fmt.Sprintf(`<p><img src="data:%s;base64,%s" alt="telegram image"/></p>`, mimeType, m.PhotoBase64))

      
        256
        325
         

      
        257
        326
         	return app.FeedEntry{

      
        258
        327
         		Title:       fmt.Sprintf("🖼️ [%s]", updated.Format("2006-01-02")),

      ···
        286
        355
         	}

      
        287
        356
         	return m.Text

      
        288
        357
         }

      
        
        358
        +

      
        
        359
        +func normalizeMessageText(text string) string {

      
        
        360
        +	text = strings.ReplaceAll(text, "\r\n", "\n")

      
        
        361
        +	return strings.ReplaceAll(text, "\r", "\n")

      
        
        362
        +}

      
        
        363
        +

      
        
        364
        +func preserveLineBreaks(text string) string {

      
        
        365
        +	if !strings.Contains(text, "\n") {

      
        
        366
        +		return text

      
        
        367
        +	}

      
        
        368
        +	return strings.ReplaceAll(text, "\n", "<br/>")

      
        
        369
        +}

      
        
        370
        +

      
        
        371
        +func messagePhotos(m *Message) []PhotoAttachment {

      
        
        372
        +	if m == nil {

      
        
        373
        +		return nil

      
        
        374
        +	}

      
        
        375
        +	if len(m.PhotoAttachments) > 0 {

      
        
        376
        +		out := make([]PhotoAttachment, len(m.PhotoAttachments))

      
        
        377
        +		copy(out, m.PhotoAttachments)

      
        
        378
        +		return out

      
        
        379
        +	}

      
        
        380
        +	if m.PhotoBase64 == "" {

      
        
        381
        +		return nil

      
        
        382
        +	}

      
        
        383
        +	mimeType := m.PhotoMIMEType

      
        
        384
        +	if mimeType == "" {

      
        
        385
        +		mimeType = "image/jpeg"

      
        
        386
        +	}

      
        
        387
        +	return []PhotoAttachment{{Base64: m.PhotoBase64, MIMEType: mimeType}}

      
        
        388
        +}

      
M sources/telegram/telegram_test.go
···
        28
        28
         	}

      
        29
        29
         }

      
        30
        30
         

      
        
        31
        +func TestFeedEntryFromMessageWithMultipleImages(t *testing.T) {

      
        
        32
        +	msg := &Message{

      
        
        33
        +		MessageID: 55,

      
        
        34
        +		Caption:   "multi image",

      
        
        35
        +		Date:      time.Date(2026, 4, 23, 12, 15, 0, 0, time.UTC).Unix(),

      
        
        36
        +		PhotoAttachments: []PhotoAttachment{

      
        
        37
        +			{Base64: "YWJj", MIMEType: "image/png"},

      
        
        38
        +			{Base64: "ZGVm", MIMEType: "image/jpeg"},

      
        
        39
        +		},

      
        
        40
        +	}

      
        
        41
        +

      
        
        42
        +	entry := feedEntryFromMessage(msg)

      
        
        43
        +	if !strings.Contains(entry.Content, `src="data:image/png;base64,YWJj"`) {

      
        
        44
        +		t.Fatalf("expected first image data URI in image entry: %s", entry.Content)

      
        
        45
        +	}

      
        
        46
        +	if !strings.Contains(entry.Content, `src="data:image/jpeg;base64,ZGVm"`) {

      
        
        47
        +		t.Fatalf("expected second image data URI in image entry: %s", entry.Content)

      
        
        48
        +	}

      
        
        49
        +}

      
        
        50
        +

      
        31
        51
         func TestFeedEntryFromMessageTextOnly(t *testing.T) {

      
        32
        52
         	msg := &Message{

      
        33
        53
         		MessageID: 11,

      ···
        39
        59
         	is.Equal(t, "plain text", entry.Title)

      
        40
        60
         	is.Equal(t, "", entry.ContentType)

      
        41
        61
         	is.Equal(t, "plain text", entry.Content)

      
        
        62
        +}

      
        
        63
        +

      
        
        64
        +func TestFeedEntryFromMessagePreservesNewlines(t *testing.T) {

      
        
        65
        +	msg := &Message{

      
        
        66
        +		MessageID: 12,

      
        
        67
        +		Text:      "line 1\nline 2",

      
        
        68
        +		Date:      time.Date(2026, 4, 22, 19, 38, 0, 0, time.UTC).Unix(),

      
        
        69
        +	}

      
        
        70
        +

      
        
        71
        +	entry := feedEntryFromMessage(msg)

      
        
        72
        +	is.Equal(t, "html", entry.ContentType)

      
        
        73
        +	if !strings.Contains(entry.Content, "line 1<br/>line 2") {

      
        
        74
        +		t.Fatalf("expected line breaks preserved in content: %s", entry.Content)

      
        
        75
        +	}

      
        42
        76
         }

      
        43
        77
         

      
        44
        78
         func TestFeedEntryFromMessageLinkifiesAndAddsAtomLinks(t *testing.T) {