andykais/grob

simple fetch utility with local caching

grob's Issues

composable scrapers

pinterest post

# grobber.yaml
name: "pinterest post"
regex: "https://pinterest.com/post/.*"
url_pattern: "https://pinterest.com/post/:post_id"
permissions:
  - 'https://pinterest.com/.*'
  - 'https://pinterest.s3.com/.*'

export async function grobber(grob, metadata) {
  const post_html = grob.fetch('https://pinterest.com/post/{{post_id}}')
  const post = grob.parse(post_html_text, 'html')
  metadata.title = post.query('.title')
  metadata.tags.push(...post.query('.tags', { force_structure: 'array' }))
  metadata.description = post.query('.description')

  const media_url = post.query('img.media', { attr: 'src' }) ?? post.query('video.source', { attr: 'src' })
  metadata.filepath = await grob.save(media_url)
}

pinterest user pins

# grobber.yaml
name: "pinterest user pins"
regex: "https://pinterest.com/user/.*"
url_pattern: "https://pinterest.com/user/:user_id"
permissions:
  - 'https://pinterest.com/.*'

export async function grobber(grob, metadata) {
  // TBD (show sequences too?)
}