Getting Search Volume with Google Adwords API (TargetingIdeaService) - new Java library

Google provides an API to get AdWords data, but there is a little amount of examples of its usage. I'll show you simple example how to get demand (search volume) data for specific words using TargetingIdeaService. In previous post I showed how to do it using the old library, in this post we'll use the new library.

Here is the example:

package loader;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import com.google.api.ads.adwords.axis.factory.AdWordsServices;
import com.google.api.ads.adwords.axis.v201302.cm.Language;
import com.google.api.ads.adwords.axis.v201302.cm.Location;
import com.google.api.ads.adwords.axis.v201302.cm.Money;
import com.google.api.ads.adwords.axis.v201302.cm.Paging;
import com.google.api.ads.adwords.axis.v201302.o.Attribute;
import com.google.api.ads.adwords.axis.v201302.o.AttributeType;
import com.google.api.ads.adwords.axis.v201302.o.IdeaType;
import com.google.api.ads.adwords.axis.v201302.o.LanguageSearchParameter;
import com.google.api.ads.adwords.axis.v201302.o.LocationSearchParameter;
import com.google.api.ads.adwords.axis.v201302.o.LongAttribute;
import com.google.api.ads.adwords.axis.v201302.o.MoneyAttribute;
import com.google.api.ads.adwords.axis.v201302.o.RelatedToQuerySearchParameter;
import com.google.api.ads.adwords.axis.v201302.o.RequestType;
import com.google.api.ads.adwords.axis.v201302.o.SearchParameter;
import com.google.api.ads.adwords.axis.v201302.o.StringAttribute;
import com.google.api.ads.adwords.axis.v201302.o.TargetingIdea;
import com.google.api.ads.adwords.axis.v201302.o.TargetingIdeaPage;
import com.google.api.ads.adwords.axis.v201302.o.TargetingIdeaSelector;
import com.google.api.ads.adwords.axis.v201302.o.TargetingIdeaServiceInterface;
import com.google.api.ads.adwords.axis.v201302.o.Type_AttributeMapEntry;
import com.google.api.ads.adwords.lib.client.AdWordsSession;
import com.google.api.ads.common.lib.auth.ClientLoginTokens;
import com.google.api.ads.common.lib.conf.ConfigurationLoadException;
import com.google.api.ads.common.lib.exception.ValidationException;
import com.google.api.client.googleapis.auth.clientlogin.ClientLoginResponseException;

public class Adwords {
    public static void main(String[] args) throws ClientLoginResponseException, IOException, ValidationException,
            ConfigurationLoadException {
        String[] locationNames = new String[] { "Paris", "Quebec", "Spain", "Deutschland" };
        String clientLoginToken = new ClientLoginTokens.Builder().forApi(ClientLoginTokens.Api.ADWORDS)
                .fromFile("adwords.properties").build().requestToken();
        AdWordsSession session = new AdWordsSession.Builder().fromFile("adwords.properties")
                .withClientLoginToken(clientLoginToken).build();
        AdWordsServices adWordsServices = new AdWordsServices();
        String[] keywords = getKeywords();

        TargetingIdeaServiceInterface targetingIdeaService = adWordsServices.get(session,
                TargetingIdeaServiceInterface.class);

        TargetingIdeaSelector selector = new TargetingIdeaSelector();

        selector.setRequestType(RequestType.STATS);
        selector.setIdeaType(IdeaType.KEYWORD);

        selector.setRequestedAttributeTypes(new AttributeType[] { AttributeType.KEYWORD_TEXT, AttributeType.SEARCH_VOLUME, AttributeType.AVERAGE_CPC });

        Language language = new Language();
        language.setId(1000L);

        // Countrycodes
        // http://code.google.com/apis/adwords/docs/appendix/countrycodes.html
        Location location = new Location();
        location.setId(2840L);

        RelatedToQuerySearchParameter relatedToQuerySearchParameter = new RelatedToQuerySearchParameter();
        relatedToQuerySearchParameter.setQueries(keywords);

        LocationSearchParameter locationSearchParameter = new LocationSearchParameter();
        locationSearchParameter.setLocations(new Location[] { location });

        LanguageSearchParameter languageSearchParameter = new LanguageSearchParameter();
        languageSearchParameter.setLanguages(new Language[] { language });

        selector.setSearchParameters(new SearchParameter[] { relatedToQuerySearchParameter, locationSearchParameter,
                languageSearchParameter // if not provided locationSearchParameter, languageSearchParameter then result
                                        // is global
        });

        selector.setLocaleCode("US");

        Paging paging = new Paging();
        paging.setStartIndex(0);
        paging.setNumberResults(keywords.length);
        selector.setPaging(paging);

        TargetingIdeaPage page = targetingIdeaService.get(selector);
        if (page.getEntries() != null && page.getEntries().length > 0) {
            for (TargetingIdea targetingIdea : page.getEntries()) {
                Map<AttributeType, Attribute> data = toMap(targetingIdea.getData());
                String kwd = ((StringAttribute) data.get(AttributeType.KEYWORD_TEXT)).getValue();
                Long monthlySearches = ((LongAttribute) data.get(AttributeType.SEARCH_VOLUME)).getValue();
                Money avgCpc = ((MoneyAttribute) data.get(AttributeType.AVERAGE_CPC)).getValue();
                
                System.out.println(kwd + ", " + monthlySearches + ", " + avgCpc.getMicroAmount() / 1000000.0);
            }
        }
    }
    
    public static String[] getKeywords() {
        //Put your keywords here
        return null;
    }
    
    public static Map<AttributeType, Attribute> toMap(Type_AttributeMapEntry[] data) {
        Map<AttributeType, Attribute> result = new HashMap<AttributeType, Attribute>();
        for (Type_AttributeMapEntry entry: data) {
            result.put(entry.getKey(), entry.getValue());
        }
        return result;        
    }
}

Read more...

Getting Top Search Queries report with Google Webmaster Tools API

Google has a good and free tool that is very useful for search engine optimization analysis. I'm talking about Google Webmaster Tools. It's possible to download the report from Google WMT using API. It's an example how to do it for Top Queries report.

At first you need to download gdata-webmastertools-2.0.jar and gdata-client-1.0.jar. You can take them from http://gdata-java-client.googlecode.com/files/gdata-src.java-1.47.1.zip (there in the lib folder you'll find the jars).

Then you'll need to place them into you PATH. I will use maven. At first I will install these jars:

mvn install:install-file -Dfile=gdata-webmastertools-2.0.jar -DgroupId=com.google.gdata -DartifactId=gdata-webmastertools -Dversion=2.0 -Dpackaging=jar -DgeneratePom=true
mvn install:install-file -Dfile=gdata-client-1.0.jar -DgroupId=com.google.gdata -DartifactId=gdata-client -Dversion=1.0 -Dpackaging=jar -DgeneratePom=true

After this I will add dependencies to my pom.xml:

<dependency>
    <groupId>com.google.gdata</groupId>
    <artifactId>gdata-client</artifactId>
    <version>1.0</version>
</dependency>
<dependency>
    <groupId>com.google.gdata</groupId>
    <artifactId>gdata-webmastertools</artifactId>
    <version>2.0</version>
</dependency>

Also we'll need to parse JSON. I will use Jackson library:

<dependency>
    <groupId>org.codehaus.jackson</groupId>
    <artifactId>jackson-mapper-asl</artifactId>
    <version>1.8.5</version>
</dependency>

And finally we are ready to get the report from Google WMT API. (In this example I also used Apache Commons and log4j but it's not necessary, so you can get rid off those dependencies)

package loader;

import com.google.gdata.client.Service.GDataRequest;
import com.google.gdata.client.Service.GDataRequest.RequestType;
import com.google.gdata.client.webmastertools.WebmasterToolsService;
import com.google.gdata.data.OutOfLineContent;
import com.google.gdata.data.webmastertools.SitesEntry;
import com.google.gdata.data.webmastertools.SitesFeed;
import com.google.gdata.util.*;
import org.apache.log4j.Logger;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
import org.apache.commons.lang.time.DateUtils;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Map;

public class GoogleWMTClient {
    private final static Logger LOGGER = Logger.getLogger(GoogleWMTClient.class);

    private final static ArrayList<String> STATISTIC_TYPE = new ArrayList<String>();
    static {
        STATISTIC_TYPE.add("ALL");
        // STATISTIC_TYPE.add("WEB");
        // STATISTIC_TYPE.add("IMAGE");
        // STATISTIC_TYPE.add("VIDEO");
        // STATISTIC_TYPE.add("MOBILE_SMARTPHONE");
        // STATISTIC_TYPE.add("MOBILE_RESTRICT");
    }
    private final static String GOOGLE_HOST = "www.google.com";
    private final static String DOWNLOAD_LIST_URL_PART = "/webmasters/tools/downloads-list?hl=%s&siteUrl=%s";
    private final static String SITES_FEED_URL_PART = "/webmasters/tools/feeds/sites/";    
    private final static String DATE_FORMAT = "yyyyMMdd";
    private final static String APPLICATION_NAME = "JavaDevTips";


    public static void main(String[] args) {
        pullData(Constants.GWMT_URL, Constants.GWMT_LANGUAGE_CODE, new Date());
    }
    
    public static void pullData(String url, String languageCode, Date endDate) {
        LOGGER.info("Download GWMT data for endDate: " + endDate + " and url: " + url);
        try {
            WebmasterToolsService service = initService(Constants.ADWORDS_USER, Constants.ADWORDS_PASSWORD);
            // used for deletion of newly created SitesEntry
            boolean newEntry = false;
            SitesEntry entry = findSitesEntry(service, url);
            if (entry == null) {
                newEntry = true;
                try {
                    entry = insertSiteEntry(service, url);
                } catch (ServiceForbiddenException ex) {
                    LOGGER.error(ex, ex);
                }
            }
            downloadReports(service, entry, endDate, languageCode);

            if (newEntry) {
                deleteSiteEntry(service, url);
            }
        } catch (ServiceException e) {
            LOGGER.error(e, e);
        } catch (IOException e) {
            LOGGER.error(e, e);
        }
    }

    public static WebmasterToolsService initService(String userName, String password) throws AuthenticationException {
        WebmasterToolsService service = new WebmasterToolsService(APPLICATION_NAME);
        service.setUserCredentials(userName, password);
        return service;
    }

    private static SitesEntry findSitesEntry(WebmasterToolsService service, String siteUrl) throws IOException,
            ServiceException {
        siteUrl = correctSiteUrl(siteUrl);
        LOGGER.info("Trying to find SitesEntry for " + siteUrl);
        SitesFeed sitesResultFeed = service.getFeed(getGoogleUrl(SITES_FEED_URL_PART), SitesFeed.class);
        for (SitesEntry entry : sitesResultFeed.getEntries()) {
            if (entry.getTitle().getPlainText().equals(siteUrl)) {
                LOGGER.info("SitesEntry is found");
                return entry;
            }
        }
        LOGGER.info("SitesEntry for " + siteUrl + " not found");
        return null;
    }

    private static URL getGoogleUrl(String path) throws MalformedURLException {
        return new URL("https://" + GOOGLE_HOST + path);
    }

    private static String correctSiteUrl(String siteUrl) {
        siteUrl = siteUrl.trim();
        if (!siteUrl.endsWith("/")) {
            siteUrl += "/";
        }
        if (!siteUrl.startsWith("http")) {
            siteUrl = "http://" + siteUrl;
        }
        return siteUrl;
    }

    private static void downloadReports(WebmasterToolsService service, SitesEntry entry, Date endDate,
            String languageCode) throws IOException, ServiceException {
        LOGGER.info("Downloading reports for " + entry.getTitle().getPlainText());
        Date startDate = DateUtils.addDays(endDate, (-1) * Constants.DATA_PERIOD);
        ObjectMapper mapper = new ObjectMapper();
        InputStream inputStream = getQueryInputStream(service, entry, languageCode);
        if (inputStream == null) {
            LOGGER.error("Empty InputStream");
            return;
        }
        Map<String, Object> map = mapper.readValue(inputStream, new TypeReference<Map<String, Object>>() {
        });
        if (map != null) {
            String fileName = null;
            SimpleDateFormat sdf = new SimpleDateFormat(DATE_FORMAT);            
            for (String prop : STATISTIC_TYPE) {
                StringBuilder sbPath = new StringBuilder((String) map.get("TOP_QUERIES")).append("&prop=" + prop)
                        .append("&db=" + sdf.format(startDate))
                        .append("&de=" + sdf.format(endDate));

                fileName = "gwmt_" + sdf.format(endDate) + prop + ".csv" ;
                OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(fileName), "UTF-8");
                boolean hasData = downloadData(service, sbPath.toString(), out);
                if (!hasData) {
                    LOGGER.info("File contains no data. Deleting.");
                    new File(fileName).delete(); // if the file contain no data we delete it
                }
                out.close();
            }
        }
    }

    private static boolean downloadData(WebmasterToolsService service, String path, OutputStreamWriter out)
            throws IOException, ServiceException {
        LOGGER.info("Downloading data for " + path);
        String data;
        URL url = getGoogleUrl(path);
        GDataRequest req = service.createRequest(RequestType.QUERY, url, ContentType.TEXT_PLAIN);
        req.execute();
        BufferedReader in = new BufferedReader(new InputStreamReader(req.getResponseStream()));
        if (in.readLine() != null) {
            while ((data = in.readLine()) != null) {
                out.write(data + "\n");
            }
            return true;
        } else {
            return false;
        }
    }

    private static InputStream getQueryInputStream(WebmasterToolsService service, SitesEntry entry, String lang)
            throws IOException, ServiceException {
        URL url = getGoogleUrl(String.format(DOWNLOAD_LIST_URL_PART, lang, entry.getTitle().getPlainText()));
        GDataRequest req = service.createRequest(RequestType.QUERY, url, ContentType.JSON);
        try {
            req.execute();
            return req.getResponseStream();
        } catch (RedirectRequiredException e) {
            LOGGER.error(e, e);
        }
        return null;
    }

    private static SitesEntry insertSiteEntry(WebmasterToolsService myService, String siteUrl) throws IOException,
            ServiceException {
        siteUrl = correctSiteUrl(siteUrl);
        SitesEntry entry = new SitesEntry();
        OutOfLineContent content = new OutOfLineContent();
        content.setUri(siteUrl);
        entry.setContent(content);
        LOGGER.info("Adding SitesEntry for  " + siteUrl);
        return myService.insert(getGoogleUrl(SITES_FEED_URL_PART), entry);
    }

    private static void deleteSiteEntry(WebmasterToolsService myService, String siteUrl) throws IOException,
            ServiceException {
        siteUrl = correctSiteUrl(siteUrl);
        String siteId = URLEncoder.encode(siteUrl, "UTF-8");
        URL feedUrl = new URL(getGoogleUrl(SITES_FEED_URL_PART) + siteId);
        SitesEntry entry = myService.getEntry(feedUrl, SitesEntry.class);
        LOGGER.info("Deleting SitesEntry for " + siteUrl);
        entry.delete();
    }
}

Read more...

Getting Search Volume with Google Adwords API (TargetingIdeaService) - old Java library

Google provides an API to get AdWords data, but there is a little amount of examples of its usage. I'll show you simple example how to get demand (search volume) data for specific words using TargetingIdeaService. This is example of old library, see the example of usage the new library.

Here is the example:

package loader;

import java.io.IOException;
import java.util.Map;

import javax.xml.rpc.ServiceException;

import common.Constants;

import com.google.api.adwords.lib.AdWordsService;
import com.google.api.adwords.lib.AdWordsUser;
import com.google.api.adwords.lib.AuthToken;
import com.google.api.adwords.lib.AuthTokenException;
import com.google.api.adwords.lib.utils.MapUtils;
import com.google.api.adwords.v201209.cm.Language;
import com.google.api.adwords.v201209.cm.Location;
import com.google.api.adwords.v201209.cm.Paging;
import com.google.api.adwords.v201209.o.Attribute;
import com.google.api.adwords.v201209.o.AttributeType;
import com.google.api.adwords.v201209.o.IdeaType;
import com.google.api.adwords.v201209.o.LanguageSearchParameter;
import com.google.api.adwords.v201209.o.LocationSearchParameter;
import com.google.api.adwords.v201209.o.LongAttribute;
import com.google.api.adwords.v201209.o.RelatedToQuerySearchParameter;
import com.google.api.adwords.v201209.o.RequestType;
import com.google.api.adwords.v201209.o.SearchParameter;
import com.google.api.adwords.v201209.o.StringAttribute;
import com.google.api.adwords.v201209.o.TargetingIdea;
import com.google.api.adwords.v201209.o.TargetingIdeaPage;
import com.google.api.adwords.v201209.o.TargetingIdeaSelector;
import com.google.api.adwords.v201209.o.TargetingIdeaServiceInterface;

public class GoogleAdwordsClient {


    private static AdWordsUser getAdWordsUser() {
        try {
            AdWordsUser user = new AdWordsUser(Constants.ADWORDS_USER, Constants.ADWORDS_PASSWORD, null, null,
                    LoaderConstants.ADWORDS_DEVELOPER_TOKEN, false);
            if (user.getRegisteredAuthToken() == null) {
                user.setAuthToken(new AuthToken(user.getEmail(), user.getPassword()).getAuthToken());
            }
            return user;
        } catch (AuthTokenException e) {
            throw new RuntimeException(e);
        }
    }

    private static String[] getKeywords() {
        //some logic to return array of keywords
    }


    public static void main(String[] args) throws ServiceException, IOException {
        String[] keywords = getKeywords();
        AdWordsUser user = getAdWordsUser();

        TargetingIdeaServiceInterface targetingIdeaService = user
                .getService(AdWordsService.V201209.TARGETING_IDEA_SERVICE);

        TargetingIdeaSelector selector = new TargetingIdeaSelector();
       
       
        selector.setRequestType(RequestType.STATS);
        selector.setIdeaType(IdeaType.KEYWORD);

        selector.setRequestedAttributeTypes(new AttributeType[] {
                AttributeType.KEYWORD_TEXT,
                AttributeType.SEARCH_VOLUME,
        });

        Language language = new Language();
        language.setId(1000L);

        // Countrycodes
        // http://code.google.com/apis/adwords/docs/appendix/countrycodes.html
        Location location = new Location();
        location.setId(2840L);

        RelatedToQuerySearchParameter relatedToQuerySearchParameter = new RelatedToQuerySearchParameter();
        relatedToQuerySearchParameter.setQueries(keywords);
       

        LocationSearchParameter locationSearchParameter = new LocationSearchParameter();
        locationSearchParameter.setLocations(new Location[]{location});
       
        LanguageSearchParameter languageSearchParameter = new LanguageSearchParameter();
        languageSearchParameter.setLanguages(new Language[]{language});
       
        selector.setSearchParameters(new SearchParameter[] { relatedToQuerySearchParameter
                , locationSearchParameter, languageSearchParameter //if not provided locationSearchParameter, languageSearchParameter then result is global
                });

        selector.setLocaleCode("US");

        Paging paging = new Paging();
        paging.setStartIndex(0);
        paging.setNumberResults(keywords.length);
        selector.setPaging(paging);

        TargetingIdeaPage page = targetingIdeaService.get(selector);
        if (page.getEntries() != null && page.getEntries().length > 0) {
            for (TargetingIdea targetingIdea : page.getEntries()) {
                Map<AttributeType, Attribute> data = MapUtils.toMap(targetingIdea.getData());
                String kwd = ((StringAttribute) data.get(AttributeType.KEYWORD_TEXT)).getValue();
                Long monthlySearches = ((LongAttribute) data.get(AttributeType.SEARCH_VOLUME)).getValue();
                
                System.out.println(kwd + ": " + monthlySearches);
            }
        }
    }
}

Read more...

Whitespaces collapse in Richfaces

It's well known that by default sequence of whitespaces in HTML will collapse to a single whitespace. To fix it it's sufficient to set CSS white-space Property:

white-space:pre-wrap;

But what to do when you use Richafaces (Ajax4jsf) and have set this CSS property correctly but nevertheless whitespaces keep collapsing?

According to RichFaces Developer Guide:

RichFaces uses a filter for a correction of code received on an Ajax request. In case of a "regular" JSF request a browser makes correction independently. In case of Ajax request in order to prevent layout destruction it's needed to use a filter, because a received code could differ from a code validated by a browser and a browser doesn't make any corrections.

Thus Richfaces parser do correct our HTML code, but the parser does not know anything about our CSS and thus here is the cause of the issue.

In Richfaces there are 2 types of parsers:

1. Tidy is the default parser. It is similar to usual browser parsing. It's the slowest one from Richfaces parsers. On my opinion it should be used only with complicated markup
2. Neko is less stricted and so it works faster

So to fix our issue it would be sufficient to change our parser from default (Tidy) to Neko. We should add according filter in web.xml:

<context-param>
   <param-name>org.ajax4jsf.xmlparser.ORDER</param-name>
   <param-value>NEKO</param-value>
</context-param>
<context-param>
   <param-name>org.ajax4jsf.xmlparser.NEKO</param-name>
    <param-value>.*\..*</param-value>
</context-param>

If necessary you can configure filters flexibly for different pages (example)

Read more...

Cannot run Spring application: BeanDefinitionParsingException

When you try to run Spring based application you can see the exception

org.springframework.beans.factory.parsing.BeanDefinitionParsingException: Unable to locate Spring NamespaceHandler for XML schema namespace

The problem may be caused by not having all neccessary Spring jars or by jar conflicts.

First of all check that you have all jars you need. Here is the list of possible jars with explanations for Spring 3.0.5 (the original of this list of dependencies is Spring Blog):

<!-- Shared version number properties -->
<properties>
    <org.springframework.version>3.0.5.RELEASE</org.springframework.version>
</properties>
 
<!--
    Core utilities used by other modules.
    Define this if you use Spring Utility APIs (org.springframework.core.*/org.springframework.util.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-core</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Expression Language (depends on spring-core)
    Define this if you use Spring Expression APIs (org.springframework.expression.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-expression</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Bean Factory and JavaBeans utilities (depends on spring-core)
    Define this if you use Spring Bean APIs (org.springframework.beans.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-beans</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Aspect Oriented Programming (AOP) Framework (depends on spring-core, spring-beans)
    Define this if you use Spring AOP APIs (org.springframework.aop.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-aop</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Application Context (depends on spring-core, spring-expression, spring-aop, spring-beans)
    This is the central artifact for Spring's Dependency Injection Container and is generally always defined
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-context</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Various Application Context utilities, including EhCache, JavaMail, Quartz, and Freemarker integration
    Define this if you need any of these integrations
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-context-support</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Transaction Management Abstraction (depends on spring-core, spring-beans, spring-aop, spring-context)
    Define this if you use Spring Transactions or DAO Exception Hierarchy
    (org.springframework.transaction.*/org.springframework.dao.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-tx</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    JDBC Data Access Library (depends on spring-core, spring-beans, spring-context, spring-tx)
    Define this if you use Spring's JdbcTemplate API (org.springframework.jdbc.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-jdbc</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Object-to-Relation-Mapping (ORM) integration with Hibernate, JPA, and iBatis.
    (depends on spring-core, spring-beans, spring-context, spring-tx)
    Define this if you need ORM (org.springframework.orm.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-orm</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Object-to-XML Mapping (OXM) abstraction and integration with JAXB, JiBX, Castor, XStream, and XML Beans.
    (depends on spring-core, spring-beans, spring-context)
    Define this if you need OXM (org.springframework.oxm.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-oxm</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Web application development utilities applicable to both Servlet and Portlet Environments
    (depends on spring-core, spring-beans, spring-context)
    Define this if you use Spring MVC, or wish to use Struts, JSF, or another web framework with Spring (org.springframework.web.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-web</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Spring MVC for Servlet Environments (depends on spring-core, spring-beans, spring-context, spring-web)
    Define this if you use Spring MVC with a Servlet Container such as Apache Tomcat (org.springframework.web.servlet.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-webmvc</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Spring MVC for Portlet Environments (depends on spring-core, spring-beans, spring-context, spring-web)
    Define this if you use Spring MVC with a Portlet Container (org.springframework.web.portlet.*)
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-webmvc-portlet</artifactId>
  <version>${org.springframework.version}</version>
</dependency>
 
<!--
    Support for testing Spring applications with tools such as JUnit and TestNG
    This artifact is generally always defined with a 'test' scope for the integration testing framework and unit testing stubs
-->
<dependency>
  <groupId>org.springframework</groupId>
  <artifactId>spring-test</artifactId>
  <version>${org.springframework.version}</version>
  <scope>test</scope>
</dependency>

If this does not help then try Maven Shade plugin to solve the most probable conflicts:

<plugin>
 <groupId>org.apache.maven.plugins</groupId>
 <artifactId>maven-shade-plugin</artifactId>
 <version>1.4</version>
 <executions>
  <execution>
   <phase>package</phase>
   <goals>
    <goal>shade</goal>
   </goals>
   <configuration>
    <transformers>
     <transformer
      implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
      <resource>META-INF/spring.handlers</resource>
     </transformer>
     <transformer
      implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
      <resource>META-INF/spring.schemas</resource>
     </transformer>
    </transformers>
   </configuration>
  </execution>
 </executions>
</plugin>

Read more...

Work with annotations in Spring Framework

Annotations are a very useful feature in Java programming language and if you use Spring Framework than it can be also very easy to create annotation processor.

I expect you to be familiar with basics of annotations (read here). In this tutorial we'll create annotation for SLF4J. Here is the annotation code:

package uay.log;

import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface Log {

}

Now let's create annotation processor using Spring Framework capabilities:

package uay.log;

import java.lang.reflect.Field;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.config.BeanPostProcessor;
import org.springframework.stereotype.Component;
import org.springframework.util.ReflectionUtils;
import org.springframework.util.ReflectionUtils.FieldCallback;

@Component
public class LogProcessor implements BeanPostProcessor {
 public Object postProcessAfterInitialization(Object bean, String beanName)
            throws BeansException {
        return bean;
    }

    public Object postProcessBeforeInitialization(final Object bean, String beanName)
            throws BeansException {
        ReflectionUtils.doWithFields(bean.getClass(), new FieldCallback() {
            public void doWith(Field field) throws IllegalArgumentException, IllegalAccessException {
                if (field.getAnnotation(Log.class) != null) {
                    Logger log = LoggerFactory.getLogger(bean.getClass());
                    field.setAccessible(true);
                    field.set(bean, log);
                    field.setAccessible(false);
                }
            }
        });
        return bean;
    }
}

And it's that easy. Now we can use this annotation in our code:

@Log
Logger log;

Read more...

Quickstart with SLF4J

In this tutorial we will configure SLF4J logging with log4j implementation.

SLF4J is a facade of abstractions for various logging frameworks(log4j, java.util.logging, commons logging, logback). It is really easy to swap between logging frameworks when you use SLF4J. Here is what you have to add to your Maven pom.xml to use SLF4J with log4j implementaion:

<properties>
 <slf4j.version>1.6.6</slf4j.version>
</properties>
<dependencies>    
 <!-- Logging -->
 <dependency>
  <groupId>log4j</groupId>
  <artifactId>log4j</artifactId>
  <version>1.2.17</version>
 </dependency>
 <dependency>
  <groupId>org.slf4j</groupId>
  <artifactId>slf4j-api</artifactId>
  <version>${slf4j.version}</version>
 </dependency>
 <dependency>
  <groupId>org.slf4j</groupId>
  <artifactId>slf4j-log4j12</artifactId>
  <version>${slf4j.version}</version>
 </dependency>
</dependencies>

Run

mvn install

Now we need to configure log4j as usual. There are two ways to do it: via xml or via properties file. Let's use properties file. We need to create log4j.properties in the classpath. Here is the code of the file:

# Root logger option
log4j.rootLogger=INFO, file, stdout
 
# Direct log messages to a log file
log4j.appender.file=org.apache.log4j.DailyRollingFileAppender
log4j.appender.file.DatePattern = '.'yyyy-MM-dd
log4j.appender.file.Append = true
log4j.appender.file.File=logs/main.log
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n
 
# Direct log messages to stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{dd-MM-yy HH:mm:ss,SSS} %5p [%c{1}:%M:%L] - %m%n

According to this configuration we defined the lowest log priority to INFO and set the output to Console and to the logs/main.log file(logs/main.log file will be rolled by days automatically).

Now we can test how the logging works:

package uay.test;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class App 
{
    private static Logger log = LoggerFactory.getLogger(App.class);
    public static void main( String[] args )
    {
     log.info("test");
    }
}

If we run the code the Console output would be:

01-12-12 23:12:24,840  INFO [App:main:11] - test

The file output would be:

23:12:24,840  INFO App:11 - test

Thus, we have configured the code to use log4j via SLF4J. As a further improvement of code you should think about using annotations to make the code look like this:

@Log
private Logger log;

This tutorial will definitely help you with it. Later I will show how to implement this annotation even easier using Spring Framework.

Read more...

Execute scripts in multiple databases

Have you ever needed to execute the same script on several databases? I have met such need and developed a simple algorithm for it. This algorithm I'd like to share.

At first I tried to develop it using only JDBC but unfortunately it has the really big limitation: it cannot execute at once several calls, which is pretty common for SQL scripts. So I decided to run scripts merely from a command line.

Here is the code:

package uay.loader;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.log4j.Logger;

public class BulkScriptLoader {

    private static final String   DB_PREFIX     = "ua";
    private static final String[] DB_SKIP = { "ua_1", "ua_2" };
    private static final String   FOLDER_NAME   = "scripts";
    private static final String   FILE_EXT      = ".sql";
    private static final String   URL           = HOST + ":" + PORT;
    private static final String   USER_NAME     = "xxx";
    private static final String   PASSWORD      = "xxx";

    private final static Logger   LOGGER        = Logger.getLogger(BulkScriptLoader.class);

    public static void main(String[] args) {
        Connection connection;
        try {
            connection = DriverManager.getConnection(URL, USER_NAME, PASSWORD);
        } catch (SQLException e1) {
            LOGGER.error(e1);
            throw new RuntimeException(e1);
        }

        List<String> dbNames = getDbNames(connection);
        File[] scriptFiles = getScriptFiles();
        for (File file : scriptFiles) {
            LOGGER.info("Work with " + file.getName());
            BufferedReader bf;
            StringBuilder statement = new StringBuilder();
            try {
                bf = new BufferedReader(new FileReader(file));
                String line = "";
                while ((line = bf.readLine()) != null) {
                    statement.append(line).append("\n");
                }
            } catch (IOException e) {
                LOGGER.error(e, e);
            }
            if (statement.toString().length() > 0) {
                execute(connection, dbNames, statement.toString(), URL, USER_NAME, PASSWORD);
            }
        }
    }

    public static File[] getScriptFiles() {
        File folder = new File(FOLDER_NAME);
        if (folder.exists() && folder.isDirectory()) {
            return folder.listFiles(new FileFilter() {
                @Override
                public boolean accept(File pathname) {
                    return pathname.getName().toLowerCase().endsWith(FILE_EXT.toLowerCase());
                }
            });
        }
        return null;
    }

    public static void execute(Connection connection, List<String> dbNames,
            String generalStatement, String url, String userName, String password) {
        for (String dbName : dbNames) {
            String cmdCommand = String.format("mysql -u %s -p%s -h %s -P %s -D %s",userName, password, 
              parseHost(url), parsePort(url), dbName);
            LOGGER.info(cmdCommand);
            try {
                Process process = Runtime.getRuntime().exec(cmdCommand);
                Writer w = new OutputStreamWriter(process.getOutputStream());
                // System.out.println(statement);
                w.write(generalStatement);
                w.flush();
                w.close();
            } catch (IOException e) {
                LOGGER.error(e, e);
            }
        }
    }

    public static String parseHost(String url) {
        return url.substring(0, (url.indexOf(":") > 0 ? url.indexOf(":"): url.length()))

    }

    public static String parsePort(String url) {
        Pattern p = Pattern.compile("(\\d\\d\\d\\d)");
        Matcher m = p.matcher(url);
        if (m.find()) {
            return m.group(0);
        }
        return "3306";
    }

    public static List<String> getDbNames(Connection connection) {
        List<String> dbNames = new ArrayList<String>();
        try {
            PreparedStatement preparedStatement = connection.
               prepareStatement("select schema_name from information_schema.schemata");
            if (preparedStatement.execute()) {
                ResultSet rs = preparedStatement.getResultSet();
                while (rs.next()) {
                    String dbName = rs.getString(1);
                    if (dbName.startsWith(DB_PREFIX) && !Arrays.asList(DB_SKIP).contains(dbName)) {
                        dbNames.add(dbName);
                    }
                }
            }
        } catch (SQLException e) {
            LOGGER.error(e, e);
        }
        return dbNames;
    }
}

As you can see it's really simple. All scripts that I'd like to execute I place in 'scripts' folder with '.sql' extension. I configure database connection, prefix of databases that I want to update and specify the list of databases that I want to skip with this update.

Read more...

JBoss remote debugging

As all you well know debugging is very important part of the development process. In this post I will show you how to configure your JBoss AS to allow debugging deployed applications.

We need to configure JBoss's bin/run.conf file. Add the following line at the end:

JAVA_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,address=8787,server=y,suspend=n $JAVA_OPTS"

This options mean:

• -Xdebug asks JBoss to work with debugging support
• -Xrunjdwp loads JDWP (Java Debug Wire Protocol). This option has its own options:
  1. transport=dt_socket means that sockets will be used for transport
  2. address=8787 means the address(in this case it's the local machine's port 8787) where the socket will be opened
  3. server=y if it is set to 'y' then it means that JBoss will listen for debugger to attach; if it is set to 'n' then it means that JBoss will attach to the debugger at the specified address
  4. suspend=n if it is set to 'y' then it means that JBoss will be launched in the suspended mode and will stay suspended until the debugger is connected

You may also want to check how to debug remote application with Eclipse

Read more...

LDAP Authentication and Search

LDAP is a very widespread way for authentication. In this post I would like to show you how to connect to LDAP server, authenticate user and perform search.

We will use only standard Java classes. So we don't need any dependency. In the example you can see my configurations. Yours may differ (pay attention to Context.SECURITY_PRINCIPAL). Here is the code:

import java.util.Properties;

import javax.naming.Context;
import javax.naming.NamingEnumeration;
import javax.naming.NamingException;
import javax.naming.directory.Attribute;
import javax.naming.directory.Attributes;
import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult;
import javax.naming.ldap.InitialLdapContext;

public class LdapTest {
    
    public static void processRequest(InitialLdapContext ctx, String userContext, String filter, String attribute) {
        SearchControls searchControls = new SearchControls();
        searchControls.setSearchScope(SearchControls.SUBTREE_SCOPE);

        NamingEnumeration<SearchResult> searchResults;
        try {
            searchResults = ctx.search(userContext, filter, searchControls);
            
            while (searchResults.hasMoreElements()) {
                SearchResult sr = searchResults.next();
                Attributes attributes = sr.getAttributes();
                Attribute a = attributes.get(attribute);
                if (a != null) {
                    String attrValue = (a.get().toString());
                    System.out.println(attrValue);
                } else {
                    System.out.println("Cannot get data");
                }
            }
        } catch (NamingException e) {
            e.printStackTrace();
        }
    }
        
    private static InitialLdapContext initialiaseLdapContext(String server, int port, String username, String password, String contextDN) {
        boolean initialised = false; 
        Properties properties = new Properties();
        properties.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory");
        properties.put(Context.PROVIDER_URL, "ldap://" + server + ":"+ port);
        properties.put(Context.SECURITY_AUTHENTICATION, "simple");
        properties.put(Context.SECURITY_PRINCIPAL, "uid=" + username + "," + contextDN); //YOU MAY NEED TO CHANGE CODE HERE
        properties.put(Context.SECURITY_CREDENTIALS, password);

        InitialLdapContext ctx = null;
        try {
            // Create initial context
            ctx = new InitialLdapContext(properties, null);
            initialised = true;
        } catch (NamingException e) {
            initialised = false;
        } finally {
            if (initialised) {
                System.out.println("Initialization success");
            } else {
                System.out.println("Initialization fail");
            }
        }
        return ctx;
    }

    public static void main(String[] args) {
        String contextDN = "dc=test,dc=com";
        InitialLdapContext ctx = initialiaseLdapContext("localhost", 389, "test@test.com", "test" , contextDN);
        try {
            if (ctx != null) {
                processRequest(ctx, contextDN, "(uid=qwertTest@test.com)", "street");
                ctx.close();
            }
        } catch (NamingException e) {
            e.printStackTrace();
        }
    }
}

Read more...

Reading XML files using XPath

Nowadays declarative programming has took a well-deserved place in imperative languages. In Java we can declare what we want in annotations or in XML. The main plus of XML is that we can reconfigure our application in one place and update it without rebuilding.

There a lot of ways to read data from XML. The most common ones are: XPath, XSLT and XQuery.

• XPath consists of path expressions, conditions and some XPath functions
• XSLT(Extensible Stylesheet Language (with) Transformation) consists of XPath and some transformations
• XQuery consists of XPath and a Query Language. This query language is pretty powerful but it does not have strong underlying mathematics like SQL.

In this tutorial we will develop a simple XPath XML file parser. But as you probably know, there are different kind of parsers:

• DOM(Document Object Model) parsers. DOM API creates a DOM tree in memory for a XML document
• SAX(Simple API For XML) parsers. They are event driven(as they parse documents they invoke the callback methods)

We will develop DOM parser. For this purpose we will use only standard Java classes without any additional dependencies.
As an example XML file we will use JBoss 5.1 datasource file. Here it is:

<?xml version="1.0" encoding="UTF-8"?>

<datasources>
    <xa-datasource>
        <jndi-name>DevDbDS</jndi-name>
 <!--
        <use-java-context>false</use-java-context>
 -->
 <xa-datasource-class>
  com.mysql.jdbc.jdbc2.optional.MysqlXADataSource
 </xa-datasource-class>
 <xa-datasource-property name="URL">
  URL:jdbc:mysql://localhost:3306/dev_db
 </xa-datasource-property>

        <user-name>root</user-name>
        <password>root</password>
 <!--
        <security-domain>VocdmsDSEncryptedLogon</security-domain>
 -->
    <xa-datasource-property name="characterEncoding">UTF-8</xa-datasource-property>

 <exception-sorter-class-name>org.jboss.resource.adapter.jdbc.vendor.MySQLExceptionSorter</exception-sorter-class-name>
        <min-pool-size>5</min-pool-size>
        <max-pool-size>20</max-pool-size>
        <blocking-timeout-millis>30000</blocking-timeout-millis>
        <idle-timeout-minutes>15</idle-timeout-minutes>
        <prefill>true</prefill>
        <transaction-isolation>TRANSACTION_READ_COMMITTED</transaction-isolation>
        <connection-property name="characterEncoding">UTF-8</connection-property>
        <connection-property name="autoReconnect">true</connection-property>
        <connection-property name="maxReconnects">4</connection-property>
        <connection-property name="initialTimeout">3</connection-property>
        <metadata>
            <type-mapping>mySQL</type-mapping>
        </metadata>
    </xa-datasource>
</datasources>

Next we need to remind ourselves how to write XPath expressions. You can do it here

Now finally we are ready to write the code to get some information from the datasource. In this example let's assume we need user name, password and URL(you may need this data to establish java.sql.Connection). So here is our code:

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathFactory;

import org.w3c.dom.Document;
import java.io.*;

public class XmlParser {
 //Immutable object for data storage
 private static class ConnectionInfo {
  private final String url;
  private final String userName;
  private final String password;
  
  public ConnectionInfo(String url, String userName, String password) {
   this.url = url;
   this.userName = userName;
   this.password = password;
  }
 
  public String getUrl() {
   return url;
  }
  public String getUserName() {
   return userName;
  }
  public String getPassword() {
   return password;
  } 
  @Override
  public String toString() {
   return "URL: " + url +
    "\nuserName: " + userName + 
    "\npassword: " + password;
  }
 }

 public static void main(String[] args) {
  System.out.println(parseDataSource("mysql-ds.xml"));
 }

 public static ConnectionInfo parseDataSource(String fileName) {
  try {   
      DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
      domFactory.setNamespaceAware(true); // never forget this!
      DocumentBuilder builder = domFactory.newDocumentBuilder();
      Document doc = builder.parse(new FileInputStream(fileName));

      XPathFactory factory = XPathFactory.newInstance();
      XPath xpath = factory.newXPath();
      String url = xpath.evaluate("//datasources/xa-datasource/xa-datasource-property[@name='URL']/text()", doc).trim();     
      String userName = xpath.evaluate("//datasources/xa-datasource/user-name/text()", doc).trim();
      String password = xpath.evaluate("//datasources/xa-datasource/password/text()", doc).trim();
      return new ConnectionInfo(url, userName, password);
  } catch (Exception ex) {
      ex.printStackTrace(); 
      return null;
  }
     } 
}

Read more...

Google Analytics API usage example

It's very useful in SEO business to keep track of different user's action on your site. One of the best tools for such kind of analysis is Google Analytics. Furthermore it's free and provides the API. In this post I will show you the simple example of usage of this API.

As usual let's start with the Maven dependencies. But unfortunately Google do not provide jars that we need in any repository. Thus we need to download and install them to Maven by ourselves. Google for "gdata core jar" and "gdata analytics jar". When you will have those files execute in command line following code to install these jars:

mvn install:install-file -Dfile=gdata-core-1.0.jar -DgroupId=com.google.gdata -DartifactId=gdata-core -Dversion=1.0 -Dpackaging=jar -DgeneratePom=true
mvn install:install-file -Dfile=gdata-analytics-2.1.jar -DgroupId=com.google.gdata -DartifactId=gdata-analytics -Dversion=2.1 -Dpackaging=jar -DgeneratePom=true

Add to pom.xml following Maven dependencies:

<dependency>
    <groupId>com.google.gdata</groupId>
    <artifactId>gdata-core</artifactId>
    <version>1.0</version>
</dependency>
<dependency>
    <groupId>com.google.gdata</groupId>
    <artifactId>gdata-analytics</artifactId>
    <version>2.1</version>
</dependency>

Also you may need Guava(which is Google's library with different basic and universal functions.

<dependency>
    <groupId>com.google.guava</groupId>
    <artifactId>guava</artifactId>
    <version>10.0.1</version>
</dependency>

After maven fetches us our new dependencies we may continue and develop Java source code for Google Analytics API data fetching.

public static DataFeed getDataFeed(String dimensions, String metrics, String userName, String password, 
        String tableId, String startDate, String endDate, int startIndex) throws IOException, ServiceException  {
    System.out.println("Building data feed");
    // Configure GA API.
    AnalyticsService as = new AnalyticsService("My application name");

    // Client Login Authorization.
    as.setUserCredentials(userName, password);
    DataQuery query = new DataQuery(new URL("https://www.google.com/analytics/feeds/data"));
    query.setIds(tableId);
    query.setDimensions(dimensions);
    query.setMetrics(metrics);
    query.setSort("-" + metrics);
    query.setMaxResults(10000);        
    query.setStartDate(startDate);
    query.setEndDate(endDate);
    query.setStartIndex(startIndex);
    URL url = query.getUrl();
    
    System.out.println("URL: " + url.toString());

    // Send our request to the Analytics API and wait for the results to
    // come back.
    DataFeed feed = as.getFeed(url, DataFeed.class);
    return feed;
}

As a parameters you should provide

• dimensions and metrics (you can get both from Google Analytics API Reference)
• Google Analytics credentials (user name and password)
• Google Analytics Profile ID(also called table ID)
• date range for quering
• start index(We need to use start index because Google Analytics service won't return result set with more then 10000 records)

Let's save data that we got to CSV file using opencsv:

public static void saveAnalyticsFile(DataFeed feed, String fileName) {
 System.out.println("Saving to file");
    try {
        CSVWriter writer = new CSVWriter(new FileWriter(fileName), ',' ,CSVWriter.DEFAULT_QUOTE_CHARACTER, '\\');
        
        DataEntry singleEntry = feed.getEntries().get(0);
        List<String> feedDataNames = new ArrayList<String>();
        List<String> dataRowValues = new ArrayList<String>(LoaderConstants.GA_FILE_HEADER.length);

        // Put all the dimension and metric names into an array.
        for (Dimension dimension : singleEntry.getDimensions()) {
            feedDataNames.add(dimension.getName());
        }
        for (Metric metric : singleEntry.getMetrics()) {
            feedDataNames.add(metric.getName());
        }

        //write header
        writer.writeNext(LoaderConstants.GA_FILE_HEADER);

        for (DataEntry entry : feed.getEntries()) {         
         //assuming that the first entry is a keyword and others are numeric values
            if (!entry.stringValueOf(feedDataNames.get(0)).equals("(not provided)") && 
                    !entry.stringValueOf(feedDataNames.get(0)).equals("(not set)")) {
             dataRowValues.add(entry.stringValueOf(feedDataNames.get(0))); //keyword
             for (int i = 1; i < feedDataNames.size(); i++) {
                 Double d = Double.parseDouble(entry.stringValueOf(feedDataNames.get(i)));
                 String googleVal = new Long(d.longValue()).toString();
                 dataRowValues.add(googleVal); 
             }
                writer.writeNext(dataRowValues.toArray(new String[0]));
             
                dataRowValues.clear();
            }
        }
        writer.close();        
        System.out.println("Saving is done");
        
    } catch (Exception e) {
        System.out.println("Cannot save file ");
        throw new RuntimeException(e);
    }
}

And finally here is the main method:

public static void main(String[] args) {
 int startIndex = 1;
 boolean fetch = true;
 while (fetch) {
     DataFeed dataFeed = getDataFeed("ga:keyword", "ga:visits", "userName", "password", 
                "profileId", "2012-08-13", "2012-08-19", startIndex);
        CsvWriteUtils.saveAnalyticsFile(dataFeed, "test");
        startIndex += 10000; //Google Analytics Max result value
        //continue fetching until we receive all data
        if (startIndex > dataFeed.getTotalResults()) {
         fetch = false;
        }
 }
}

Read more...

AuthorityLabs Partner API

If you need to get SERP(search engine results page) rank information on some keywords, there is a very good solution - AuthorityLabs Partner API. I have successfully used it on one of my projects and I want to share with you some tips on developing functionality to work with it.

AuthorityLabs Partner API is as a REST service. The main API methods are for POSTing keywords that you want to get info on and GETting the information from POSTed keywords. There are 2 different ways of work with POST requests:

• Instant POST request(expensive)
• Delayed POST request(cheaper)

We'll work with Delayed requests.
At first we should think how are we going to work with the Web service. I have chosen to use Apache HttpComponents because it's straightforward and easy to use. To download the library I used the following Maven dependencies:

<dependency>
 <groupId>org.apache.httpcomponents</groupId>
 <artifactId>httpcore</artifactId>
 <version>4.2</version>
</dependency>
 <dependency>
 <groupId>org.apache.httpcomponents</groupId>
 <artifactId>httpclient</artifactId>
 <version>4.1.3</version>
</dependency>

Now let's develop method to POST keywords to the AuthorityLabs Partner API.

public class AuthorityLabsTest {
 private static final String AUTHORITYLABS_AUTH_TOKEN = "XXX";
 private static final String AUTHORITYLABS_URL_POST_KEYWORD = "http://api.authoritylabs.com/keywords";

 
  /**
     * Post keyword to the delayed queue of the AuthorityLabs API
     * @param keyword
     * @param engine
     * @param locale
     * @return
     */
 static boolean postKeyword(String keyword, String engine, String locale) {
  HttpClient client = new DefaultHttpClient();
  
  HttpPost post = new HttpPost(AUTHORITYLABS_URL_POST_KEYWORD);
  try {
   List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(1);
   nameValuePairs.add(new BasicNameValuePair("auth_token", AUTHORITYLABS_AUTH_TOKEN));
   nameValuePairs.add(new BasicNameValuePair("keyword",keyword));
   nameValuePairs.add(new BasicNameValuePair("engine", engine));
   nameValuePairs.add(new BasicNameValuePair("locale", locale));  
   post.setEntity(new UrlEncodedFormEntity(nameValuePairs));
   System.out.println("Post keyword; parameters - " + nameValuePairs);
 
   HttpResponse response = client.execute(post);
   BufferedReader rd = new BufferedReader(new InputStreamReader(
     response.getEntity().getContent()));
   String line = "";
   while ((line = rd.readLine()) != null) {
    System.out.println(line);
   }

  } catch (IOException e) {
   e.printStackTrace();
   return false;
  }
  return true;
 }
}

Next we need method to GET keyword info.

public class AuthorityLabsTest {
 private static final String AUTHORITYLABS_AUTH_TOKEN = "XXX";
 private static final String AUTHORITYLABS_URL_POST_KEYWORD = "http://api.authoritylabs.com/keywords";
 private static final String AUTHORITYLABS_URL_GET_RESULTS = "http://api.authoritylabs.com/keywords/get.json?keyword={0}&auth_token={1}&engine={2}&locale={3}&rank_date={4}";

 static boolean postKeyword(String keyword, String engine, String locale) {
    ...
 }
 
 /**
  * Gets JSON results from the AuthorityLabs API
  * @param keyword
  * @param engine
  * @param locale
  * @param stringPostKeywordDate
  * @return
  */
 static InputStream getKeywordResults(String keyword, String engine, String locale, String stringPostKeywordDate) {
  String urlPattern = AUTHORITYLABS_URL_GET_RESULTS;
  String url = MessageFormat.format(urlPattern, keyword, AUTHORITYLABS_AUTH_TOKEN, 
    engine, locale, stringPostKeywordDate);
  //Prepare URL (e.g. replace spaces with %20 etc)
  try {
   url = URIUtil.encodeQuery(url);
  } catch (URIException e1) {
   e1.printStackTrace();
   return null;
  }
  System.out.println("Get keyword results; url - " + url);
  
  HttpClient client = new DefaultHttpClient();
  HttpGet get = new HttpGet(url); 
  HttpResponse response;
  try {
   response = client.execute(get);
   System.out.println(response.getStatusLine());   
   return response.getEntity().getContent();
  } catch (Exception e) {
   e.printStackTrace();
  }   
  return null;
 }
}

Note that we need to encode URL in order to be able to get results. To choose correct engine and locale for that engine please check the documentation on AuthorityLabs API.

That's the core of the work with AuthorityLabs API.

Read more...

Creating mock OFX server

Nowadays more and more financial establishments use modern Web technologies to provide user with better experience. One of such technologies is OFX.

OFX stands for Open Financial Exchange, it is the format, that is based on XML, for data exchange between financial institutions. It was released in 1997 and evolved from Microsoft's Open Financial Connectivity (OFC) and Intuit's Open Exchange file formats.

In Java there are not many OFX libraries, the best one is OFX4J. Using this library you can develop both client and server. It has classes for every possible OFX (XML) element. On my opinion, the biggest flaw of this library is the bad documentation. Unfortunately, there are also not many examples online on usage of OFX4J. So here is my example that may help someone.

Let's write the simple JAX-WS OFX server. As always at first we need to download the library or specify Maven dependency:

<dependency>
 <groupId>net.sf.ofx4j</groupId>
 <artifactId>ofx4j</artifactId>
 <version>1.5</version>
</dependency>

Now we need the web-service interface:

package com.test.ofx.server.service;

import javax.activation.DataHandler;
import javax.jws.WebMethod;
import javax.jws.WebService;

@WebService
public interface OFXServerService {    
    @WebMethod
    DataHandler getResponse(DataHandler request);    
}

As you can see there was nothing difficult. Now it's time for the implementation:

package com.test.ofx.server.service.impl;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.TreeSet;
import java.util.UUID;

import javax.activation.DataHandler;
import javax.jws.WebService;

import net.sf.ofx4j.domain.data.ApplicationSecurity;
import net.sf.ofx4j.domain.data.RequestEnvelope;
import net.sf.ofx4j.domain.data.RequestMessage;
import net.sf.ofx4j.domain.data.RequestMessageSet;
import net.sf.ofx4j.domain.data.ResponseEnvelope;
import net.sf.ofx4j.domain.data.ResponseMessageSet;
import net.sf.ofx4j.domain.data.banking.AccountType;
import net.sf.ofx4j.domain.data.banking.BankAccountDetails;
import net.sf.ofx4j.domain.data.banking.BankStatementRequestTransaction;
import net.sf.ofx4j.domain.data.banking.BankStatementResponse;
import net.sf.ofx4j.domain.data.banking.BankStatementResponseTransaction;
import net.sf.ofx4j.domain.data.banking.BankingResponseMessageSet;
import net.sf.ofx4j.domain.data.common.Status;
import net.sf.ofx4j.domain.data.common.Status.Severity;
import net.sf.ofx4j.io.AggregateMarshaller;
import net.sf.ofx4j.io.AggregateUnmarshaller;
import net.sf.ofx4j.io.OFXParseException;
import net.sf.ofx4j.io.OFXWriter;
import net.sf.ofx4j.io.v2.OFXV2Writer;

import com.sun.xml.ws.util.ByteArrayDataSource;
import com.test.ofx.server.service.OFXServerService;

@WebService(endpointInterface = "com.test.ofx.server.service.OFXServerService", name = "ofxWebService")
public class OFXServerServiceImpl implements OFXServerService {
    
    @Override
    public DataHandler getResponse(DataHandler request) {
        try {
            //parse request            
            AggregateUnmarshaller<requestenvelope> unmarshaller = new AggregateUnmarshaller<requestenvelope>(
                    RequestEnvelope.class);
            RequestEnvelope requestEnvelope = unmarshaller.unmarshal(request.getInputStream());

            ResponseEnvelope responseEnvelope = getResponse(requestEnvelope);

            //prepare response to send
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            OFXWriter writer = new OFXV2Writer(baos);
            AggregateMarshaller marshaller = new AggregateMarshaller();
            
            marshaller.marshal(responseEnvelope, writer);
            writer.close();
            DataHandler dataHandler = new DataHandler(new ByteArrayDataSource(baos.toByteArray(),
                    "application/octet-stream"));
            return dataHandler;
        } catch (IOException e) {
            throw new RuntimeException(e);
        } catch (OFXParseException e) {
            throw new RuntimeException(e);
        }
    }

    private ResponseEnvelope getResponse(RequestEnvelope requestEnvelope) {
        //declare variables to generate response
        ResponseEnvelope responseEnvelope = new ResponseEnvelope();            
        BankingResponseMessageSet responseMessageSet = new BankingResponseMessageSet();
        List<bankstatementresponsetransaction> transactions = new ArrayList<bankstatementresponsetransaction>();
        BankStatementResponse bankStatementResponse = new BankStatementResponse();
        BankAccountDetails bankAccountDetails;
        Random rand = new Random();

        //generate response transactions
        for (RequestMessageSet requestMessageSet : requestEnvelope.getMessageSets()) {
            for (RequestMessage requestMessage : requestMessageSet.getRequestMessages()) {
                //specify bank account details
                bankAccountDetails = new BankAccountDetails();
                bankAccountDetails
                        .setAccountNumber(((BankStatementRequestTransaction) requestMessage)
                                .getMessage().getAccount().getAccountNumber()
                                + "-" + rand.nextInt());
                bankAccountDetails.setBankId("Bank ID");
                bankAccountDetails.setAccountType(AccountType.CHECKING);
                
                //specify and add bank statment response transaction
                bankStatementResponse.setAccount(bankAccountDetails);
                BankStatementResponseTransaction bankStatementResponseTransaction = new BankStatementResponseTransaction();
                bankStatementResponseTransaction.setMessage(bankStatementResponse);
                Status status = new Status();
                status.setSeverity(Severity.INFO);        
                bankStatementResponseTransaction.setStatus(status);
                bankStatementResponseTransaction.setUID(UUID.randomUUID().toString());
                transactions.add(bankStatementResponseTransaction);
            }
        }

        //create response envelope
        responseMessageSet.setStatementResponses(transactions);
        responseEnvelope.setMessageSets(new TreeSet<responsemessageset>());
        responseEnvelope.getMessageSets().add(responseMessageSet);
        responseEnvelope.setSecurity(ApplicationSecurity.TYPE1);
        responseEnvelope.setUID(UUID.randomUUID().toString());
        return responseEnvelope;
    }
}

There are many classes in the library and at first the code may seem pretty incomprehensible. But you have to remember that almost all this variety of classes do nothing more than just wrap XML element. Every time you are not sure about meaning of the current operation feel free to look the library code. It is not very complicated.

By the way at first I've tried to get RequestEnvelope and send ResponseEnvelope in the web service, but I've stumbled upon implementation of ResponseEnvelope. It is dependable on some class that do not have public default constructor(UnknownStatusCode). And due to limitations of JAX-WS it was impossible to create stubs or use the common interface with the client.

That's all about the server part. In the next article we'll create the OFX client.

Read more...

Full working runtime annotation example

Annotations are metadata(data about data) that allows you to keep an additional information right in your code. Annotations have been introduced into Java in 5th edition.

Before annotations almost all configuration data had to be kept in XML format. And as the application grew, the number and complexity of its XML-files had been increasing exponentially. They called it XML hell.

And now when we have annotations we can keep the configuration of the code right with it. Besides annotations are tested and verified by the compiler.

But the use of annotations is not limited to configuration, annotations allow you to get rid of the boilerplate code (e.g. cross-cutting concerns such as transactions and security).

The boon of annotations can be very easily noted using Hibernate Annotation package, Spring 2.5 and newer, Seam. But how do they write the annotations of their own?

So here is the definition of our annotation @Testing:

package com.test.annotation;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface Testing { }

As you can see this definition is not much unlike of the definition of an interface. And really it compiles to the class file as any other Java interface. As you can see the annotation definition requires @Target and @Retention annotations:

• @Target defines where you can use your annotation. In this example it can be used only with methods. Here are all possible values and places where you can use annotations:
• TYPE - class, interface (including annotation type), or enum declaration 
• FIELD - field declaration (includes enum constants)
• METHOD - method declaration
• PARAMETER - parameter declaration
• CONSTRUCTOR - constructor declaration
• LOCAL_VARIABLE - local variable declaration
• ANNOTATION_TYPE - annotation type declaration 
• PACKAGE - package declaration
• @Retention defines when the annotation can be available:
• SOURCE - in the source code (discarded by the compiler)
• CLASS - in the class files (not retained at run time). This is the default behavior.
• RUNTIME - at run time (can be read reflectively)

Often annotations contain some values (as @Target and @Retention above). These values are represented as interface methods in the definition. If an annotation does not have any element than it is called marker annotation.

We will use proxy pattern to allow our processor intercept the calls to the methods annotated with @Testing. Java provides implementation of the proxy with the java.lang.reflect.Proxy. It is very powerful tool but unfortunately it does not provide possibility to create proxies just for the class, you have to provide the list of interfaces to create the proxy (if you need to proxy classes and not just interfaces then you should use CGLib).

So let's create the annotation processor:

package com.test.annotation.processor;

import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;

import com.test.annotation.Testing;

public class TestingInvocationHandler implements InvocationHandler {
 private Object proxied;
 
 public TestingInvocationHandler(Object proxied) {
  this.proxied = proxied; } 

 @Override
 public Object invoke(Object proxy, Method method, Object[] args)
   throws Throwable {
  Method m = proxied.getClass().getMethod(method.getName(), method.getParameterTypes());
  if (m.isAnnotationPresent(Testing.class)) {
   System.out.println("\tIn the annotation processor");   
  } 
  return method.invoke(proxied, args);
 }
}

The annotation processor implements InvocationHandler interface. This interface is responsible for handling the proxy object invocation.
As you can see we only print additional line before invoking the original method.

Here is the helper class with the static method to get the proxy object:

package com.test.annotation.processor;
import java.lang.reflect.Proxy;

public class TestingProxy {
 
 public static Object getNewProxy(Object proxied, Class<?> interfaze) {
  Object proxy = Proxy.newProxyInstance(
      TestingInvocationHandler.class.getClassLoader(),
      new Class[] {interfaze}, 
      new TestingInvocationHandler(proxied));
  return proxy;
 }

}

Let's define the interface

package com.test.service;

public interface Observable {
 void print1();
 
 void print2(); 
}

and the class of the proxied objects

package com.test.service;

import com.test.annotation.Testing;
import com.test.annotation.processor.TestingProxy;

public class PrintService implements Observable {
 private static Observable instance;
 private PrintService(){
  
 }
 public static Observable getInstance() {
  if (instance == null) {   
   instance = (Observable) TestingProxy.getNewProxy(new PrintService(),
     Observable.class);   
  }
  return instance;
 }
 
 @Testing
 public void print1() {
  System.out.println("1 - Text in annotated method");
 }
 
 public void print2() {
  System.out.println("2 - Just ususal text");
 }
}

This class is implemented as a simple singleton and uses the proxy object in getInstance() method. Also this class implements both methods of the interface. One of this methods is annotated and the other is not.

Finally let's create the class with the main method:

package com.test;

import com.test.service.Observable;
import com.test.service.PrintService;

public class Application {
 public static void main(String[] args) {
  Observable printService = PrintService.getInstance();  
  printService.print1();  
  printService.print2();
 }
}

That's all.

Read more...

Java FTP client

If you are looking for Java FTP client I suggest you to look at Apache Commons Net library. There is a class FTPClient. It is very easy to use and has all the functionality you may want to have.
To use this library we have to download it from the official site or add maven dependency:

<dependency>
    <groupId>commons-net</groupId>
    <artifactId>commons-net</artifactId>
    <version>3.0.1</version>
</dependency>

After this if we want to upload the file to the server we should do something like this:

public static boolean uploadFile(File file, String workingDirectory, String fileName) {
  FTPClient client = new FTPClient();
  try {
    client.connect(FTP_HOST, FTP_PORT);
    // After connection attempt, we have to check the reply code to
    // verify success.
    int reply = client.getReplyCode();
    if (!FTPReply.isPositiveCompletion(reply)) {
      System.err.println("FTP server refused connection.");
      return false;
    }
  
    client.login(FTP_USERNAME, FTP_PASSWORD);
    client.changeWorkingDirectory(workingDirectory);
    client.storeFile(fileName, new FileInputStream(file));
    client.logout();
    return true;
  } catch (SocketException e) {
    throw new RuntimeException(e);
  } catch (IOException e) {
    throw new RuntimeException(e);
  } finally {
    if (client.isConnected()) {
      try {
        client.disconnect();
      } catch (IOException ioe) {
        // swallow exception
        System.err.println("Cannot disconnect from the host");
      }
    }
  } 
}

If you need to download, delete or append file it would be pretty similar.
To get listing of directories you may need to enter local passive mode. Here is an example:

public static void printFiles(String workingDirectory) {
  FTPClient client = new FTPClient();
  try {
    client.connect(FTP_HOST, FTP_PORT);   
    // After connection attempt, we have to check the reply code to
    // verify success.
    int reply = client.getReplyCode();
    if (!FTPReply.isPositiveCompletion(reply)) {
      System.err.println("FTP server refused connection.");
    }
  
    client.enterLocalPassiveMode();
    client.login(FTP_USERNAME, FTP_PASSWORD);
    FTPFile[] files = client.listFiles(workingDirectory);
     
    for (FTPFile file : files) {
      System.out.println(file.getName());
    }
    client.logout();
  } catch (SocketException e) {
    throw new RuntimeException(e);
  } catch (IOException e) {
    throw new RuntimeException(e);
  } finally {
    if (client.isConnected()) {
      try {
        client.disconnect();
      } catch (IOException ioe) {
        // swallow exception
        System.err.println("Cannot disconnect from the host");
      }
    }
  }
}

It was easy wasn't it? To learn more see the documentation.

Read more...

Recursively unzip archive in memory

In addition to the previous post about recursive unzipping you may need to unzip archive in memory. In this situation the following code will help you:

package com.test;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

public class Unzipper {
 private final static int BUFFER_SIZE = 2048;
 private final static String ZIP_EXTENSION = ".zip";
 
 public static void main(String[] args) throws IOException {
  File f = new File("/home/anton/test/test.zip");
  FileInputStream fis = new FileInputStream(f);
  BufferedInputStream bis = new BufferedInputStream(fis);
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  BufferedOutputStream bos = new BufferedOutputStream(baos);
  byte[] buffer = new byte[BUFFER_SIZE];
  while (bis.read(buffer, 0, BUFFER_SIZE) != -1) {
   bos.write(buffer);
  }
  bos.flush();
  bos.close();
  bis.close();
  List<ByteArrayOutputStream> listFiles =  unzip(baos);
 }

 public static List<ByteArrayOutputStream> unzip(
   ByteArrayOutputStream zippedFileOS) {
  try {
   ZipInputStream inputStream = new ZipInputStream(
     new BufferedInputStream(new ByteArrayInputStream(
       zippedFileOS.toByteArray())));
   ZipEntry entry;

   List<ByteArrayOutputStream> result = new ArrayList<ByteArrayOutputStream>();
   while ((entry = inputStream.getNextEntry()) != null) {
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    System.out.println("\tExtracting entry: " + entry);
    int count;
    byte data[] = new byte[BUFFER_SIZE];

    if (!entry.isDirectory()) {
     BufferedOutputStream out = new BufferedOutputStream(
       outputStream, BUFFER_SIZE);
     while ((count = inputStream.read(data, 0, BUFFER_SIZE)) != -1) {
      out.write(data, 0, count);
     }
     out.flush();
     out.close();
     // recursively unzip files
     if (entry.getName().toLowerCase().endsWith(ZIP_EXTENSION)) {
      result.addAll(unzip(outputStream));
     } else {
      result.add(outputStream);
     }
    }
   }
   inputStream.close();
   return result;
  } catch (Exception e) {
   throw new RuntimeException(e);
  }
 }
}

You may also want to check how to recursively unzip archive to file

Read more...

Recursively unzip archive

Once I had to unzip some difficult zip archive that consisted of inner zip archives, directories and files. I have googled for examples, though there were some solutions I didn't find the perfect one for me. So I decided to develop my own unzipper. Here is it, feel free to use it:

package com.test;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

public class Unzipper {  
    private final static int BUFFER_SIZE = 2048;
    private final static String ZIP_FILE = "/home/anton/test/test.zip";
    private final static String DESTINATION_DIRECTORY = "/home/anton/test/";
    private final static String ZIP_EXTENSION = ".zip";
 
    public static void main(String[] args) {
     System.out.println("Trying to unzip file " + ZIP_FILE); 
        Unzipper unzip = new Unzipper();  
        if (unzip.unzipToFile(ZIP_FILE, DESTINATION_DIRECTORY)) {
         System.out.println("Succefully unzipped to the directory " 
             + DESTINATION_DIRECTORY);
        } else {
         System.out.println("There was some error during extracting archive to the directory " 
             + DESTINATION_DIRECTORY);
        }
    } 

 public boolean unzipToFile(String srcZipFileName,
   String destDirectoryName) {
  try {
   BufferedInputStream bufIS = null;
   // create the destination directory structure (if needed)
   File destDirectory = new File(destDirectoryName);
   destDirectory.mkdirs();

   // open archive for reading
   File file = new File(srcZipFileName);
   ZipFile zipFile = new ZipFile(file, ZipFile.OPEN_READ);

   //for every zip archive entry do
   Enumeration<? extends ZipEntry> zipFileEntries = zipFile.entries();
   while (zipFileEntries.hasMoreElements()) {
    ZipEntry entry = (ZipEntry) zipFileEntries.nextElement();
    System.out.println("\tExtracting entry: " + entry);

    //create destination file
    File destFile = new File(destDirectory, entry.getName());

    //create parent directories if needed
    File parentDestFile = destFile.getParentFile();    
    parentDestFile.mkdirs();    
    
    if (!entry.isDirectory()) {
     bufIS = new BufferedInputStream(
       zipFile.getInputStream(entry));
     int currentByte;

     // buffer for writing file
     byte data[] = new byte[BUFFER_SIZE];

     // write the current file to disk
     FileOutputStream fOS = new FileOutputStream(destFile);
     BufferedOutputStream bufOS = new BufferedOutputStream(fOS, BUFFER_SIZE);

     while ((currentByte = bufIS.read(data, 0, BUFFER_SIZE)) != -1) {
      bufOS.write(data, 0, currentByte);
     }

     // close BufferedOutputStream
     bufOS.flush();
     bufOS.close();

     // recursively unzip files
     if (entry.getName().toLowerCase().endsWith(ZIP_EXTENSION)) {
      String zipFilePath = destDirectory.getPath() + File.separatorChar + entry.getName();

      unzipToFile(zipFilePath, zipFilePath.substring(0, 
              zipFilePath.length() - ZIP_EXTENSION.length()));
     }
    }
   }
   bufIS.close();
   return true;
  } catch (Exception e) {
   e.printStackTrace();
   return false;
  }
 } 
}  

I have placed some comments inline so now it seems pretty straightforward, but if you have some questions feel free to ask them.
You may also want to check how to recursively unzip archive in memory

Read more...

How to make correct classloading using JBoss AS 5.1

After I have added new functionality to my project I've spent a lot of time trying to make it work with JBoss AS 5.1 . The problem was that JBoss AS 5.1 has some libraries in the package with itself (look $JBOSS_HOME/lib, $JBOSS_HOME/lib/endorsed, $JBOSS_HOME/common/lib) and I wanted to use different version then provided with JBoss. But let's tell about everything consequently.

At first I tried to do nothing specific and have just specified my maven dependencies with compile scope. And of course I got exception:

java.lang.ClassCastException: org.apache.xerces.jaxp.DocumentBuilderFactoryImpl cannot be cast to javax.xml.parsers.DocumentBuilderFactory
    at javax.xml.parsers.DocumentBuilderFactory.newInstance(Unknown Source)
    at org.apache.log4j.xml.DOMConfigurator.doConfigure(DOMConfigurator.java:694)
    at org.apache.log4j.xml.DOMConfigurator.doConfigure(DOMConfigurator.java:618)
    at org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionConverter.java:470)
    at org.apache.log4j.LogManager.<clinit>(LogManager.java:122)
    ... 82 more

At first I got flabbergasted. What does this exception mean? I don't even use Apache Xerces in my project.
So that means one of libraries I use is dependent on it. Then I tried to write separate mock project for this newly added libraries. And it worked fine. Then it means JBoss is involved with this exception. And of course it is. JBoss had the xercesImpl.jar in $JBOSS_HOME/lib/endorsed.

So here we have version conflict. What can we do in this situation. There are some solutions to this situation:

1. Remove all conflicting jars from your application (make all conflicting maven dependencies in provided scope). But in my opinion, this is bad idea, because
• it would make the application dependent on Application Server
• it may result in a lot of rework on currently good working code 
2. Change JBoss AS libraries with the correct versions (all conflicting maven dependencies should be in provided scope). But it has another problems:
• you have to reconfigure your server; if you develop in team then all your teammates have to do the same with their working environment;
• it may result in LinkageError. In my case it meant that QName class was already loaded by boot Classloader and it means that I cannot use it in my classes(loaded by another Classloader). If you're interested about this error see link1 and link2.

• java.lang.LinkageError: loader constraint violation: when resolving method
      "org.apache.axis2.description.AxisOperation.setName(Ljavax/xml/namespace/QName;)V"
      the class loader (instance of org/jboss/classloader/spi/base/BaseClassLoader) of 
      the current class, some/path/to/my/class, and the class loader 
      (instance of <bootloader>) for resolved class, 
      org/apache/axis2/description/AxisOperation, have different Class objects for 
      the type javax/xml/namespace/QName used in the signature

3. Configure JBoss AS classloading for the war with jboss-classloading.xml in your war/ear/jar. This is the best solution on my opinion. Because your application would be still Application Server independent(other servers won't pay attention to this additional file) and does not require additional configuration of Application Server.

So let's discuss the third solution. At first you have to create new jboss-classloading.xml in the correct location:

• If your application is WAR then place it in /WEB-INF/
• If your application is EAR or JAR then place it in /META-INF/

And put configuration there. Here is the configuration that worked for me:

<classloading xmlns="urn:jboss:classloading:1.0"
    name="myApp.war"
    parent-first="false"
    domain="DefaultDomain"
    top-level-classloader="true"
    parent-domain="Ignored"
    export-all="NON_EMPTY"
    import-all="true">
</classloading>

The meaning of attributes:

• name - usually the name of war/ear/jar
• parent-first (true/false) - the classloader should load at first everything from your war/ear/jar and then from parent(in case of war/jar within ear it would mean ear, otherwise it's JBossAS)
• domain - classloading domain, if it already exists then you will add your application there
• top-level-classloader (true/false) - allows you to take part in parent classloading
• parent-domain - delegates the classloading for the specified domain if the class is not available in the current domain
• export-all - exposes your classes to other applications
• import-all (true/false) - import exposes classes from other applications

That's all. In conclusion here is the links that helped me with this error:

• http://www.mastertheboss.com/jboss-application-server/223-solving-jboss-5-classloading-issues.html
• http://community.jboss.org/wiki/useJBossWebClassLoaderinJBoss5
• https://issues.jboss.org/browse/JBDEPLOY-222
• https://issues.jboss.org/browse/JBCL-125

Read more...

Reading Excel files with Apache POI(usermodel)

Recently I have stumbled upon need to read spreadsheet data from usual Excel files. After some research I have found that the most suitable solution for me is to use Apache POI. It works fine, simple to use, has a long history(10 years) and is still alive.

The POI API is full of acronyms, the meanings of which is not understandable (e.g. HSSF, XSSF, SXSSF) until you read the documentation. So let's start with some definitions:

• HSSF(Horrible SpreadSheet Format) is the POI Project's pure Java implementation of the Excel '97(-2007) file format. 
• XSSF (XML SpreadSheet Format) is the POI Project's pure Java implementation of the Excel 2007 OOXML (.xlsx) file format.
• SXSSF is an API-compatible streaming extension of XSSF to be used when very large spreadsheets have to be produced, and heap space is limited. It's built on top of XSSF. Available only since 3.8-beta3.

Before I show you some examples of work I want you to be aware of additional complexities in POI. There are different ways to read files. The API provides us with 2 capabilities, they are:

• UserModel
• UserEventModel. 

The first one is really straightforward and easy to use, but the second one is much more intricate and is based upon reading files using SAX (e. g. Apache Xerces). And by the way if you need to load big files then you should choose usereventmodel because usermodel consumes a lot of memory and it's very likely you will become the victim of the dreadful Java OutOfMemory exception. And if you need to write to SpreadSheet then you have no choice and will be forced to use usermodel. In this article we will see the usage of usermodel API for reading.

Figure 1. Spreadsheet API Feature Summary

So at first you need to download the library from official site or to configure maven dependencies. I prefer the second option, so here are dependencies (I got them from mvnrepository.com):

<dependency>
    <groupId>org.apache.poi</groupId>
    <artifactId>poi</artifactId>
    <version>3.5-FINAL</version>
    <type>jar</type>
    <scope>compile</scope>
</dependency>
<dependency>
    <groupId>org.apache.poi</groupId>
    <artifactId>poi-contrib</artifactId>
    <version>3.5-FINAL</version>
    <type>jar</type>
    <scope>compile</scope>
</dependency>
<dependency>
    <groupId>org.apache.poi</groupId>
    <artifactId>poi-ooxml</artifactId>
    <version>3.5-FINAL</version>
    <type>jar</type>
    <scope>compile</scope>
</dependency>

There are two ways to read data in usermodel:

• Read only data and no null value. It usually produces ragged arrays(depends on your data)
• Read all data (inclusive nulls)

Now let's create the code to read all data:

public List<List<String>> parseSpreadSheet(InputStream inputStream) {
    Workbook workBook = null;
    try {
        workBook = WorkbookFactory.create(inputStream);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    Sheet sheet = workBook.getSheetAt(0);    
    List<List<String>> rowHolder = new ArrayList<List<String>>();
    int cellNum = sheet.getRow(1).getLastCellNum();    

    for (int i = 0; i <= sheet.getLastRowNum(); i++) {
        Row row = sheet.getRow(i);
        List<String> cellHolder = new ArrayList<String>();

        for (int j = 0; j < row.getLastCellNum(); j++) {
            Cell cell = row.getCell(j);           
            String cellValue = parseCellValue(workBook, cell);
            cellHolder.add(cellValue);
        }

        //add empty cells to the end if required
        while (cellHolder.size() < cellNum) {
            cellHolder.add(null);                   
        }
        rowHolder.add(cellHolder);    
    }
    return rowHolder;
}

The return type of the method is Java SpreadSheet representation (selected by me). The argument of the method is InputStream of the SpreadSheet file.

I have decided to use Workbook(represents Excel file), which is the parent class of HSSFWorkbook and XSSFWorkbook. WorkbookFactory frees you from necessity to write conditions for different kinds of files (xls and xlsx), it creates appropriate descendant class from InputStream. The rest of the code is simple(The code of parseCellValue() method I will show later).

We need the method to parse cell values:

private String parseCellValue(Workbook workBook, Cell cell) {       
    FormulaEvaluator evaluator = workBook.getCreationHelper().createFormulaEvaluator();
    String cellValue = null;               
    if (cell != null) {
        switch (cell.getCellType()) {
            case Cell.CELL_TYPE_STRING:
                cellValue = cell.getRichStringCellValue().getString();
                break;
            case Cell.CELL_TYPE_NUMERIC:
                if (DateUtil.isCellDateFormatted(cell)) {
                    cellValue = cell.getDateCellValue().toString();
                } else {
                    cellValue = new Double(cell.getNumericCellValue()).toString();
                }
                break;
            case Cell.CELL_TYPE_BOOLEAN:
                cellValue = new Boolean(cell.getBooleanCellValue()).toString();
                break;
            case Cell.CELL_TYPE_FORMULA:
                cellValue = evaluator.evaluate(cell).formatAsString();
                break;
        }                   
    }
    return cellValue;
}

As you can see we should parse cell value according to its Excel format. Note POI UserAPI is pretty powerful it even allows us to evaluate formula values.

And finally here's the way you can read all data but null values. This type of reading is based upon iterators of Row and Cell:

public List<List<String>> readSpreadSheetWOnull(InputStream inputStream) {
    Workbook workBook = null;
    try {
        workBook = WorkbookFactory.create(inputStream);
        Sheet sheet = workBook.getSheetAt(SHEET_NUMBER);

        Iterator<Row> rowIter = sheet.rowIterator();

        List<List<String>> rowHolder = new ArrayList<List<String>>();
        while (rowIter.hasNext()) {
            Row row = (Row) rowIter.next();
            Iterator<Cell> cellIter = row.cellIterator();

            List<String> cellHolder = new ArrayList<String>();
            while (cellIter.hasNext()) {
                Cell cell = (Cell) cellIter.next();
                String cellValue = parseCellValue(workBook, cell);
                cellHolder.add(cellValue);
            }
            rowHolder.add(cellHolder);
        }
        return rowHolder;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

That's all. Later I will write a post about UserEventModel

Read more...