Libraries

library(rvest) # parse html
library(stringr) #string functions
library(jsonlite) # parse json
library(tidyverse) # make R fun
library(lubridate) # parse dates
library(tidytext) # text functions
library(httr) # api requests

API Key

Before we can get any data from the GroupMe API, we’ll need to get an API key. Log into your GroupMe account at dev.groupme.com. In the menu bar at the top right you will see an access token menu item.

GroupMe Access Token Menu

Click the menu item to get your key and save it as a variable in your R project.

groupme.api.key <- "iamanapikey"

Query the Data

The GroupMe API separates messaging calls from group calls. You can specify the group(s) you want messages from using group IDs, which will allow you to use the messaging API to get messages from the group(s).

First, I’ll get a list of all my groups.

connection.url <- 'https://api.groupme.com/v3/groups'
groups.query.response <- GET(connection.url,
                      query = list(
                        token = groupme.api.key
                      ))
parsed.group.response <- content(groups.query.response, "parsed")

You can explore the parsed.group.response to find the group you need if you’re only looking for one group’s messages. Instead, I’m going to get every message from all my groups using a list of the group IDs.

group.ids <- lapply(parsed.group.response$response, 
                    function(x){ x$group})

Test Getting Messages

Before trying to get every message from every group, we should test out the API with a single group.

Get your group ID.

test.group.id <- group.ids[1]

Query a group for messages.

#/groups/:group_id/messages

connection.url <- paste0("https://api.groupme.com/v3/groups/",
                         test.group.id,
                         "/messages")
query.response <- GET(connection.url,
                      query = list(
                        token = groupme.api.key
                      ))
parsed.data <- content(query.response, "parsed")

I’ll use lapply to get the fields I want out of the message.

messages <- lapply(
    parsed.data$response$messages, 
    function(x){
      as.vector(x[c("sender_id", 
                    "name",
                    "text")])
      }
    )

View the messages to see messages people wrote in that group.

head(messages)

When converting to a vector we lose our null values, which makes it hard to get the messages into a dataframe format and create a word cloud. To get around that issue I am using a function to convert any null items to NA.

get.non.null <- function(item){
  if(is.null(item)){
    return(NA)
  }
  else{
    return(item)
  }
}

Now we can convert our sample into a dataframe using the get.non.null function.

message.df <- data.frame("id"= c(NA),
                         "name" = c(NA),
                         "text" = c(NA),
                         stringsAsFactors = FALSE)
for(message in messages){
  message.vector <- c(get.non.null(message$sender_id[1]),
                      get.non.null(message$name[1]),
                      get.non.null(message$text[1]))
  # print(message.vector)
  message.df <- rbind(message.df,message.vector)
}

summary(message.df)

The GroupMe API also limits how many messages you can query at a time (default is 20). To get all of the messages from a single group we have to query multiple times.

 while(query.response$status_code == 200){
    parsed.data <- content(query.response, "parsed")
    messages <- lapply(parsed.data$response$messages,
                       function(x){x[c("id", 
                                       "sender_id", 
                                       "name", 
                                       "text")]})
    for(message in messages){
          message.vector <- c(get.non.null(message$sender_id[1]),
                              get.non.null(message$name[1]), 
                              get.non.null(message$text[1]))
    # print(message.vector)
    message.df <- rbind(message.df,message.vector)
    }
    last.message.id <- message.df$id[length(message.df$id)]
    #query again starting at the last message
    query.response <- GET(connection.url,
                      query = list(
                        token = groupme.api.key,
                        before_id = last.message.id
                      ))

  }
  #success if 304, otherwise exited on failed query
  print(paste("exited with code:", query.response$status_code))

summary(message.df)

Make a Function

Combine everything into a function so we can repeat for every group.

get.messages <-  function(group_id){
  #Connect to API for messages in the group
  connection.url <- paste0("https://api.groupme.com/v3/groups/",
                             group_id,
                             "/messages")
  #Define what we want out of a group message
  message.df <- data.frame("group_id" = c(NA), 
                           "message_id" = c(NA), 
                           "sender_id"= c(NA), 
                           "name" = c(NA), 
                           "text" = c(NA), 
                           stringsAsFactors = FALSE)
  
  #Query first 100 messages of a group
  query.response <- GET(connection.url,
                        query = list(
                          token = groupme.api.key,
                          limit = 100
                        ))
  
  #Continue querying for messages until we receive an API error
  while(query.response$status_code == 200){
    parsed.data <- content(query.response, "parsed")
    #Parse message fields
    messages <- lapply(parsed.data$response$messages,
                       function(x){x[c("id", 
                                       "sender_id", 
                                       "name", 
                                       "text")]})
    #Convert into a dataframe
    for(message in messages){
      message.vector <- c(group_id, 
                          get.non.null(message$id[1]), 
                          get.non.null(message$sender_id[1]), 
                          get.non.null(message$name[1]), 
                          get.non.null(message$text[1]))
      # print(message.vector)
      message.df <- rbind(message.df,message.vector)
    }
    #Get latest message ID
    last.message.id <- message.df$message_id[length(message.df$message_id)]
    #Query for more messages
    query.response <- GET(connection.url,
                        query = list(
                          token = groupme.api.key,
                          limit = 100,
                          before_id = last.message.id
                        ))
  }
  #success if 304, otherwise failed
  print(paste("exited with code:", query.response$status_code))
  return(message.df)
}

It’s time to actually retrieve all the data we need from GroupMe. I’m both combining all the data and saving each group to its own file.

#Create our final dataframe
all.messages.df <- data.frame("group_id"= c(NA),
                              "message_id" = c(NA),
                              "sender_id"= c(NA), 
                              "name" = c(NA), 
                              "text" = c(NA), 
                              stringsAsFactors = FALSE)

#Run our function on every group
for(id in group.ids){
  print(paste("group", id))
  #Check how long it takes to get all messages
  time <- system.time(messages.df <- get.messages(id))
  print(time)
  #save as individual rds file
  saveRDS(messages.df,
          file = paste0("../data/group_messages/",id,".rds"))
  all.messages.df <- rbind(all.messages.df, messages.df)
}
#save final dataframe
saveRDS(all.messages.df, file = "../data/all_messages.rds")

summary(all.messages.df)

##    group_id          message_id         sender_id        
##  Length:5376        Length:5376        Length:5376       
##  Class :character   Class :character   Class :character  
##  Mode  :character   Mode  :character   Mode  :character  
##      name               text          
##  Length:5376        Length:5376       
##  Class :character   Class :character  
##  Mode  :character   Mode  :character

Most Relevant Words

For this gift I wanted to stray from word frequency. Since we have a group of many people messaging, I can compare the words my giftee used with what everyone else said using Term Frequency, Inverse Document Frequency (TF-IDF).

This time we don’t start by filtering a single person’s messages, instead only removing bad messages such as links containing “.com”.

filtered.messages <- all.messages.df %>%
  filter(!str_detect(text, ".com"))

TF-IDF is created on all messages, and grouped by each person with the final function bind_tf_idf(word, sender_id, n).

group.words <- filtered.messages %>%
    unnest_tokens(word, text) %>%
    anti_join(stop_words) %>%
    count(word, sender_id, sort = TRUE)

## Joining, by = "word"

group.words.tfidf <- group.words %>% 
  bind_tf_idf(word, sender_id, n)

With wordcloud2 we can easily use the tf_idf numbers in place of word frequency. The package automatically determines how to size the words and doesn’t require whole numbers. We also won’t need to filter stop words with tf_idf, although you still can if you want to. TF-IDF adjusts for usage across everyone. Stop words, which everyone uses in the group, will have lower values. This is particularly helpful for me as my name (Will), is a stop word. We only need to filter a single person at this final step and remake the word cloud.

my.tfidf <- group.words.tfidf %>% 
  filter(sender_id == sender.id)

wordcloud.df <- my.tfidf %>%
  select(word, tf_idf)

wordcloud2(wordcloud.df)