guy176251/reddit-go
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle listings better by using custom unmarshaling

Browse Source 
Signed-off-by: Vartan Benohanian <vartanbeno@gmail.com>
This commit is contained in:
Vartan Benohanian
2020-05-03 17:31:35 -04:00
parent 460554e19e
commit 7922711d51
9 changed files with 988 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files
subreddit.go
+267 -172
View File
@@ -5,7 +5,9 @@ import (
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
)
// SubredditService handles communication with the subreddit
@@ -23,14 +25,22 @@ type SubredditService interface {
GetMineWhereModerator(ctx context.Context, opts *ListOptions) (*SubredditList, *Response, error)
GetMineWhereStreams(ctx context.Context, opts *ListOptions) (*SubredditList, *Response, error)
GetHotPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error)
GetNewPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error)
GetRisingPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error)
GetControversialPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error)
GetTopPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error)
GetHotLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error)
GetBestLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error)
GetNewLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error)
GetRisingLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error)
GetControversialLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error)
GetTopLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error)
GetSticky1(ctx context.Context, name string) (interface{}, *Response, error)
GetSticky2(ctx context.Context, name string) (interface{}, *Response, error)
// GetSticky1(ctx context.Context, name string) (interface{}, *Response, error)
// GetSticky2(ctx context.Context, name string) (interface{}, *Response, error)
Subscribe(ctx context.Context, names ...string) (*Response, error)
SubscribeByID(ctx context.Context, ids ...string) (*Response, error)
Unsubscribe(ctx context.Context, names ...string) (*Response, error)
UnsubscribeByID(ctx context.Context, ids ...string) (*Response, error)
StreamLinks(ctx context.Context, names ...string) (<-chan Link, chan<- bool, error)
}
// SubredditServiceOp implements the SubredditService interface
@@ -40,42 +50,6 @@ type SubredditServiceOp struct {
var _ SubredditService = &SubredditServiceOp{}
type subredditRoot struct {
Kind *string `json:"kind,omitempty"`
Data *Subreddit `json:"data,omitempty"`
}
type subredditRootListing struct {
Kind *string `json:"kind,omitempty"`
Data *struct {
Dist int `json:"dist"`
Roots []subredditRoot `json:"children,omitempty"`
After string `json:"after,omitempty"`
Before string `json:"before,omitempty"`
} `json:"data,omitempty"`
}
// Subreddit holds information about a subreddit
type Subreddit struct {
ID string `json:"id,omitempty"`
FullID string `json:"name,omitempty"`
Created float64 `json:"created"`
CreatedUTC float64 `json:"created_utc"`
URL string `json:"url,omitempty"`
DisplayName string `json:"display_name,omitempty"`
DisplayNamePrefixed string `json:"display_name_prefixed,omitempty"`
Title string `json:"title,omitempty"`
PublicDescription string `json:"public_description,omitempty"`
Type string `json:"subreddit_type,omitempty"`
SuggestedCommentSort string `json:"suggested_comment_sort,omitempty"`
Subscribers int `json:"subscribers"`
ActiveUserCount *int `json:"active_user_count,omitempty"`
NSFW bool `json:"over18"`
UserIsMod bool `json:"user_is_moderator"`
}
// SubredditList holds information about a list of subreddits
// The after and before fields help decide the anchor point for a subsequent
// call that returns a list
@@ -85,58 +59,14 @@ type SubredditList struct {
Before string `json:"before,omitempty"`
}
type submissionRoot struct {
Kind *string `json:"kind,omitempty"`
Data *Submission `json:"data,omitempty"`
}
type submissionRootListing struct {
Kind *string `json:"kind,omitempty"`
Data *struct {
Dist int `json:"dist"`
Roots []submissionRoot `json:"children,omitempty"`
After string `json:"after,omitempty"`
Before string `json:"before,omitempty"`
} `json:"data,omitempty"`
}
// Submission is a submitted post on Reddit
type Submission struct {
ID string `json:"id,omitempty"`
FullID string `json:"name,omitempty"`
Created float64 `json:"created"`
CreatedUTC float64 `json:"created_utc"`
Permalink string `json:"permalink,omitempty"`
URL string `json:"url,omitempty"`
Title string `json:"title,omitempty"`
Body string `json:"selftext,omitempty"`
Score int `json:"score"`
NumberOfComments int `json:"num_comments"`
SubredditID string `json:"t5_2qo4s,omitempty"`
SubredditName string `json:"subreddit,omitempty"`
SubredditNamePrefixed string `json:"subreddit_name_prefixed,omitempty"`
AuthorID string `json:"author_fullname,omitempty"`
AuthorName string `json:"author,omitempty"`
Spoiler bool `json:"spoiler"`
Locked bool `json:"locked"`
NSFW bool `json:"over_18"`
IsSelfPost bool `json:"is_self"`
Saved bool `json:"saved"`
Stickied bool `json:"stickied"`
}
// SubmissionList holds information about a list of subreddits
// LinkList holds information about a list of links
// The after and before fields help decide the anchor point for a subsequent
// call that returns a list
type SubmissionList struct {
Submissions []Submission `json:"submissions,omitempty"`
After string `json:"after,omitempty"`
Before string `json:"before,omitempty"`
// Note: not to be confused with linked lists
type LinkList struct {
Links []Link `json:"submissions,omitempty"`
After string `json:"after,omitempty"`
Before string `json:"before,omitempty"`
}
// GetByName gets a subreddit by name
@@ -220,44 +150,44 @@ var sorts = [...]string{
"top",
}
// GetHotPosts returns the hot posts
// If no subreddit names are provided, then it runs the search against /r/all
// GetHotLinks returns the hot links
// If no subreddit names are provided, then it runs the search against all those the client is subscribed to
// IMPORTANT: for subreddits, this will include the stickied posts (if any)
// PLUS the number of posts from the limit parameter (which is 25 by default)
func (s *SubredditServiceOp) GetHotPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error) {
return s.getPosts(ctx, sortHot, opts, names...)
func (s *SubredditServiceOp) GetHotLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error) {
return s.getLinks(ctx, sortHot, opts, names...)
}
// GetBestPosts returns the best posts
// If no subreddit names are provided, then it runs the search against /r/all
// GetBestLinks returns the best links
// If no subreddit names are provided, then it runs the search against all those the client is subscribed to
// IMPORTANT: for subreddits, this will include the stickied posts (if any)
// PLUS the number of posts from the limit parameter (which is 25 by default)
func (s *SubredditServiceOp) GetBestPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error) {
return s.getPosts(ctx, sortBest, opts, names...)
func (s *SubredditServiceOp) GetBestLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error) {
return s.getLinks(ctx, sortBest, opts, names...)
}
// GetNewPosts returns the new posts
// If no subreddit names are provided, then it runs the search against /r/all
func (s *SubredditServiceOp) GetNewPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error) {
return s.getPosts(ctx, sortNew, opts, names...)
// GetNewLinks returns the new links
// If no subreddit names are provided, then it runs the search against all those the client is subscribed to
func (s *SubredditServiceOp) GetNewLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error) {
return s.getLinks(ctx, sortNew, opts, names...)
}
// GetRisingPosts returns the rising posts
// If no subreddit names are provided, then it runs the search against /r/all
func (s *SubredditServiceOp) GetRisingPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error) {
return s.getPosts(ctx, sortRising, opts, names...)
// GetRisingLinks returns the rising links
// If no subreddit names are provided, then it runs the search against all those the client is subscribed to
func (s *SubredditServiceOp) GetRisingLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error) {
return s.getLinks(ctx, sortRising, opts, names...)
}
// GetControversialPosts returns the controversial posts
// If no subreddit names are provided, then it runs the search against /r/all
func (s *SubredditServiceOp) GetControversialPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error) {
return s.getPosts(ctx, sortControversial, opts, names...)
// GetControversialLinks returns the controversial links
// If no subreddit names are provided, then it runs the search against all those the client is subscribed to
func (s *SubredditServiceOp) GetControversialLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error) {
return s.getLinks(ctx, sortControversial, opts, names...)
}
// GetTopPosts returns the top posts
// If no subreddit names are provided, then it runs the search against /r/all
func (s *SubredditServiceOp) GetTopPosts(ctx context.Context, opts *ListOptions, names ...string) (*SubmissionList, *Response, error) {
return s.getPosts(ctx, sortTop, opts, names...)
// GetTopLinks returns the top links
// If no subreddit names are provided, then it runs the search against all those the client is subscribed to
func (s *SubredditServiceOp) GetTopLinks(ctx context.Context, opts *ListOptions, names ...string) (*LinkList, *Response, error) {
return s.getLinks(ctx, sortTop, opts, names...)
}
type sticky int
@@ -267,14 +197,65 @@ const (
sticky2
)
// GetSticky1 returns the first stickied post on a subreddit (if it exists)
func (s *SubredditServiceOp) GetSticky1(ctx context.Context, name string) (interface{}, *Response, error) {
return s.getSticky(ctx, name, sticky1)
// // GetSticky1 returns the first stickied post on a subreddit (if it exists)
// func (s *SubredditServiceOp) GetSticky1(ctx context.Context, name string) (interface{}, *Response, error) {
// return s.getSticky(ctx, name, sticky1)
// }
// // GetSticky2 returns the second stickied post on a subreddit (if it exists)
// func (s *SubredditServiceOp) GetSticky2(ctx context.Context, name string) (interface{}, *Response, error) {
// return s.getSticky(ctx, name, sticky2)
// }
// Subscribe subscribes to subreddits based on their name
// Returns {} on success
func (s *SubredditServiceOp) Subscribe(ctx context.Context, names ...string) (*Response, error) {
form := url.Values{}
form.Set("action", "sub")
form.Set("sr_name", strings.Join(names, ","))
return s.handleSubscription(ctx, form)
}
// GetSticky2 returns the second stickied post on a subreddit (if it exists)
func (s *SubredditServiceOp) GetSticky2(ctx context.Context, name string) (interface{}, *Response, error) {
return s.getSticky(ctx, name, sticky2)
// SubscribeByID subscribes to subreddits based on their id
// Returns {} on success
func (s *SubredditServiceOp) SubscribeByID(ctx context.Context, ids ...string) (*Response, error) {
form := url.Values{}
form.Set("action", "sub")
form.Set("sr", strings.Join(ids, ","))
return s.handleSubscription(ctx, form)
}
// Unsubscribe unsubscribes from subreddits
// Returns {} on success
func (s *SubredditServiceOp) Unsubscribe(ctx context.Context, names ...string) (*Response, error) {
form := url.Values{}
form.Set("action", "unsub")
form.Set("sr_name", strings.Join(names, ","))
return s.handleSubscription(ctx, form)
}
// UnsubscribeByID unsubscribes from subreddits based on their id
// Returns {} on success
func (s *SubredditServiceOp) UnsubscribeByID(ctx context.Context, ids ...string) (*Response, error) {
form := url.Values{}
form.Set("action", "unsub")
form.Set("sr", strings.Join(ids, ","))
return s.handleSubscription(ctx, form)
}
func (s *SubredditServiceOp) handleSubscription(ctx context.Context, form url.Values) (*Response, error) {
path := "api/subscribe"
req, err := s.client.NewPostForm(path, form)
if err != nil {
return nil, err
}
resp, err := s.client.Do(ctx, req, nil)
if err != nil {
return resp, err
}
return resp, nil
}
func (s *SubredditServiceOp) getSubreddits(ctx context.Context, path string, opts *ListOptions) (*SubredditList, *Response, error) {
@@ -288,30 +269,24 @@ func (s *SubredditServiceOp) getSubreddits(ctx context.Context, path string, opt
return nil, nil, err
}
root := new(subredditRootListing)
root := new(rootListing)
resp, err := s.client.Do(ctx, req, root)
if err != nil {
return nil, resp, err
}
if root.Data == nil {
return nil, resp, nil
l := new(SubredditList)
if root.Data != nil {
l.Subreddits = root.Data.Things.Subreddits
l.After = root.Data.After
l.Before = root.Data.Before
}
sl := new(SubredditList)
var subreddits []Subreddit
for _, child := range root.Data.Roots {
subreddits = append(subreddits, *child.Data)
}
sl.Subreddits = subreddits
sl.After = root.Data.After
sl.Before = root.Data.Before
return sl, resp, nil
return l, resp, nil
}
func (s *SubredditServiceOp) getPosts(ctx context.Context, sort sort, opts *ListOptions, names ...string) (*SubmissionList, *Response, error) {
func (s *SubredditServiceOp) getLinks(ctx context.Context, sort sort, opts *ListOptions, names ...string) (*LinkList, *Response, error) {
path := sorts[sort]
if len(names) > 0 {
path = fmt.Sprintf("r/%s/%s", strings.Join(names, "+"), sorts[sort])
@@ -327,27 +302,21 @@ func (s *SubredditServiceOp) getPosts(ctx context.Context, sort sort, opts *List
return nil, nil, err
}
root := new(submissionRootListing)
root := new(rootListing)
resp, err := s.client.Do(ctx, req, root)
if err != nil {
return nil, resp, err
}
if root.Data == nil {
return nil, resp, nil
l := new(LinkList)
if root.Data != nil {
l.Links = root.Data.Things.Links
l.After = root.Data.After
l.Before = root.Data.Before
}
sl := new(SubmissionList)
var submissions []Submission
for _, child := range root.Data.Roots {
submissions = append(submissions, *child.Data)
}
sl.Submissions = submissions
sl.After = root.Data.After
sl.Before = root.Data.Before
return sl, resp, nil
return l, resp, nil
}
// getSticky returns one of the 2 stickied posts of the subreddit
@@ -355,28 +324,154 @@ func (s *SubredditServiceOp) getPosts(ctx context.Context, sort sort, opts *List
// If it's <= 1, it's 1
// If it's >= 2, it's 2
// todo
func (s *SubredditServiceOp) getSticky(ctx context.Context, name string, num sticky) (interface{}, *Response, error) {
// type query struct {
// Num sticky `url:"num"`
// }
// func (s *SubredditServiceOp) getSticky(ctx context.Context, name string, num sticky) (interface{}, *Response, error) {
// type query struct {
// Num sticky `url:"num"`
// }
// path := fmt.Sprintf("r/%s/about/sticky", name)
// path, err := addOptions(path, query{num})
// if err != nil {
// return nil, nil, err
// }
// path := fmt.Sprintf("r/%s/about/sticky", name)
// path, err := addOptions(path, query{num})
// if err != nil {
// return nil, nil, err
// }
// req, err := s.client.NewRequest(http.MethodGet, path, nil)
// if err != nil {
// return nil, nil, err
// }
// req, err := s.client.NewRequest(http.MethodGet, path, nil)
// if err != nil {
// return nil, nil, err
// }
// root := new(submissionRootListing)
// resp, err := s.client.Do(ctx, req, root)
// if err != nil {
// return nil, resp, err
// }
// var root []rootListing
// resp, err := s.client.Do(ctx, req, &root)
// if err != nil {
// return nil, resp, err
// }
// return nil, resp, nil
return nil, nil, nil
// // test, _ := json.MarshalIndent(root, "", " ")
// // fmt.Println(string(test))
// linkRoot := new(linkRoot)
// link := root[0].Data.Children[0]
// byteValue, err := json.Marshal(link)
// if err != nil {
// return nil, resp, err
// }
// err = json.Unmarshal(byteValue, linkRoot)
// if err != nil {
// return nil, resp, err
// }
// // these are all the comments in the post
// comments := root[1].Data.Children
// var commentsRoot []commentRoot
// byteValue, err = json.Marshal(comments)
// if err != nil {
// return nil, resp, err
// }
// err = json.Unmarshal(byteValue, &commentsRoot)
// if err != nil {
// return nil, resp, err
// }
// test, _ := json.MarshalIndent(commentsRoot, "", " ")
// fmt.Println(string(test))
// for _, comment := range commentsRoot {
// if string(comment.Data.RepliesRaw) == `""` {
// comment.Data.Replies = nil
// continue
// }
// // var
// }
// return commentsRoot, resp, nil
// }
// func handleComments(comments []commentRoot) {
// for _, comment := range comments {
// if string(comment.Data.RepliesRaw) == `""` {
// comment.Data.Replies = nil
// continue
// }
// }
// }
// StreamLinks returns a channel that receives new submissions from the subreddits
// To stop the stream, simply send a bool value to the stop channel
func (s *SubredditServiceOp) StreamLinks(ctx context.Context, names ...string) (<-chan Link, chan<- bool, error) {
if len(names) == 0 {
return nil, nil, errors.New("must specify at least one subreddit")
}
submissionCh := make(chan Link)
stop := make(chan bool, 1)
go func() {
// todo: if the post with the before gets deleted, you keep getting 0 posts
var last *Timestamp
for {
select {
case <-stop:
close(submissionCh)
return
default:
sl, _, err := s.GetNewLinks(ctx, nil, names...)
if err != nil {
continue
}
var newest *Timestamp
for i, submission := range sl.Links {
if i == 0 {
newest = submission.Created
}
if last == nil {
submissionCh <- submission
continue
}
if last.Before(*submission.Created) {
submissionCh <- submission
}
}
last = newest
}
<-time.After(time.Second * 3)
fmt.Println()
}
}()
// go func() {
// var before string
// for {
// select {
// case <-stop:
// close(submissionCh)
// return
// default:
// sl, _, err := s.GetSubmissions(ctx, SortNew, &ListOptions{Before: before}, names...)
// if err != nil {
// continue
// }
// fmt.Printf("Received %d posts\n", len(sl.Submissions))
// if len(sl.Submissions) == 0 {
// continue
// }
// for _, submission := range sl.Submissions {
// submissionCh <- submission
// }
// before = sl.Submissions[0].FullID
// }
// <-time.After(time.Second * 5)
// fmt.Println()
// }
// }()
return submissionCh, stop, nil
}