Ruby wrapper for Google App Engine URL Fetch service

I am still playing with my Sinatra app, running on GAE. Just in case if do that too… and might need to fetch some external data, following is a wrapper code I’m using :

require 'java'

module UrlFetch
  
  module UF
    import java.net.URL;
    import java.net.URLEncoder;    
    import com.google.appengine.api.urlfetch.HTTPHeader
    import com.google.appengine.api.urlfetch.HTTPMethod
    import com.google.appengine.api.urlfetch.HTTPRequest
    import com.google.appengine.api.urlfetch.HTTPResponse
    import com.google.appengine.api.urlfetch.URLFetchService
    import com.google.appengine.api.urlfetch.URLFetchServiceFactory
    
    Service = URLFetchServiceFactory.getURLFetchService()
  end
  
  module InstanceMethods
    def fetch_url(options = {})
      self.class.fetch_url(options)
    end

  end

  module ClassMethods
  
    # = Fetch URL proxy
    # === Accepted options:
    # :url     - request url
    # :method  - HTTP method ('get', 'post' ..)
    # :headers - hash of request headers
    # :params  - request params. Only valid if the method is post
    #
    # === Response:
    # Rack stype responce:
    # [response_code, headers, body]
    
    def fetch_url(options = {})
      return nil unless (options[:url])
      
      url = UF::URL.new(options[:url])
      request = UF::HTTPRequest.new(url, UF::HTTPMethod.valueOf((options[:method] || 'get').upcase))
      
      options[:headers].each{|name, value| request.addHeader(UF::HTTPHeader.new(name, value))} if options[:headers] && options[:headers].is_a?(Hash)
      
      if options[:method] == 'post' && options[:params]
        payload = options[:params].collect{|name, value| "#{UF::URLEncoder.encode(name, 'UTF-8')}=#{UF::URLEncoder.encode(value, 'UTF-8')}" }
        request.setPayload(payload.to_java_bytes)
      end
      response = UF::Service.fetch(request)
      [
        response.getResponseCode,
        response.getHeaders().inject({}){|hash, header| hash[header.name] = header.value; hash },
        (String.from_java_bytes(response.getContent) if response.getContent)
      ]
    rescue => e
      [500, {}, e.to_s]
    end

  end
  
  def self.included(base)
    base.send :include, InstanceMethods
    base.send :extend,  ClassMethods
  end
  
end

I hope this is will save some typing 🙂

Here is a link to GitHub gist: git clone git://gist.github.com/111006.git gist-111006

Advertisements

Rack::Cache on Google App Engine

It’s been a few weeks now since Google announced Java support on their App Engine infrastructure. Since Java support also adds support for JRuby, it is now possible to deploy Rails or Sinatra or any other Rack based applications to Google. In my case, I am playing with Sinatra 🙂 This is an excelent step-by-step tutorial on how to get started with Sinatra App. If you running your applications on App Engine, you might be interested (at least I was) in reducing your application load by using some sort of page cache. Since Google won’t allow you to write out files, I figured using their MemCachedService might be pretty nice way to work around this issue. Ryan Tomayko has written a very nice Rack middle-ware, Rack::Cache, which provides support for cache control and validation and looked to me like a very nice solution. Following is my extension to Ryan’s work, which will allow to use Rack::Cache with Google Memcache as a meta-data and entity storage.
I have the following code in ./lib/cache.rb file:

require 'java'
require 'rack/cache'
require 'yaml'

module Cache

module MC
import com.google.appengine.api.memcache.Expiration;
import com.google.appengine.api.memcache.MemcacheService;
import com.google.appengine.api.memcache.MemcacheServiceFactory;
import com.google.appengine.api.memcache.Stats;

Service = MemcacheServiceFactory.getMemcacheService
end

module ClassMethods
def clear
MC::Service.clearAll
end

def exists?(key)
MC::Service.contains(key)
end

alias_method :contains?, :exists?

def get(key)
value = MC::Service.get(key)
YAML.load(value) if value
end

def put(key, value, ttl = nil)
expiration = ttl ? MC::Expiration.byDeltaSeconds(ttl) : nil
MC::Service.put(key, value.to_yaml, expiration)
end

def namespace
MC::Service.getNamespace
end

def namespace=(value)
MC::Service.setNamespace(value.to_s)
end

def delete(key)
MC::Service.delete(key)
end
end

module Service
extend Cache::ClassMethods
end

end

module Rack::Cache

class MetaStore

public

class GAEStore < MetaStore
attr_reader :cache

def initialize(options = {})
@cache = Cache::Service
@cache.namespace = options[:namespace] if options[:namespace]
end

def read(key)
key = hexdigest(key)
@cache.get(key) || []
end

def write(key, entries)
key = hexdigest(key)
@cache.put(key, entries)
end

def purge(key)
key = hexdigest(key)
@cache.delete(key)
end

def self.resolve(uri)
self.new(:namespace => uri.host)
end

end

GAECACHE = GAEStore
GAE = GAEStore
end

class EntityStore

public

class GAEStore < EntityStore
attr_reader :cache

def initialize(options = {})
@cache = Cache::Service
@cache.namespace = options[:namespace] if options[:namespace]
end

def exist?(key)
@cache.exists?(key)
end

def read(key)
result = @cache.get(key)
result
end

def open(key)
if data = read(key)
[data]
else
nil
end
end

def write(body)
buf = StringIO.new
key, size = slurp(body){|part| buf.write(part) }
@cache.put(key, buf.string)
[key, size]
end

def self.resolve(uri)
self.new(:namespace => uri.host)
end
end

GAECACHE = GAEStore
GAE = GAEStore

end
end

Add the following into config.ru:

……..

require ‘application’

use Rack::Cache, {:metastore => ‘gae://namespace‘, :entitystore => ‘gae://namespace‘}

run Sinatra::Application

In order to make it work properly, read Ryan’s documentation on all supported options.

I am hoping this is helpful 🙂