Pushing Data to Apache Solr Using SpringData

From past few days i was working on Apache Solr Technologies to index data in one of my projects. Today i wanted to elaborate on this concept, this post assumes that you already have Apache Solr installed on your system, if not please install the latest( > 4.1) version from here. Read the instruction from solr site make the Solr server up and running. Once this Solr is up and running we can go to Admin dashboard by hitting the following url http://localhost:8983/solr.

Now its time for us to dive into Spring coding inorder to push data to solr server. this project is written on top of Spring4, SpringData, Solr4, Maven and Eclipse Luna.

first lets create a maven project and put the below code in your pom.xml file and the project structure looks as below.

SolrProjectStructure

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>com.spark.solr</groupId>
	<artifactId>SolrSpringIntegration</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<packaging>war</packaging>
	<name>SolrSpringIntegration</name>
	<description>SolrSpringIntegration</description>
	<properties>
		<jdk.version>1.6</jdk.version>
		<spring.version>4.0.4.RELEASE</spring.version>
		<spring.security.version>3.2.3.RELEASE</spring.security.version>
		<jstl.version>1.2</jstl.version>
		<spring-data-solr.verion>1.3.0.RELEASE</spring-data-solr.verion>
	</properties>

	<dependencies>

		<!-- Spring dependencies -->
		<dependency>
			<groupId>org.springframework</groupId>
			<artifactId>spring-core</artifactId>
			<version>${spring.version}</version>
		</dependency>

		<dependency>
			<groupId>org.springframework</groupId>
			<artifactId>spring-web</artifactId>
			<version>${spring.version}</version>
		</dependency>

		<dependency>
			<groupId>org.springframework</groupId>
			<artifactId>spring-webmvc</artifactId>
			<version>${spring.version}</version>
		</dependency>

		<dependency>
			<groupId>org.springframework</groupId>
			<artifactId>spring-oxm</artifactId>
			<version>${spring.version}</version>
		</dependency>

		<dependency>
			<groupId>org.springframework</groupId>
			<artifactId>spring-jdbc</artifactId>
			<version>${spring.version}</version>
		</dependency>

		<!-- SOLR -->
		<dependency>
			<groupId>org.springframework.data</groupId>
			<artifactId>spring-data-solr</artifactId>
			<version>${spring-data-solr.verion}</version>
		</dependency>

		<!-- jstl and servlet for jsp page -->
		<dependency>
			<groupId>javax.servlet</groupId>
			<artifactId>javax.servlet-api</artifactId>
			<version>3.0.1</version>
		</dependency>
		<dependency>
			<groupId>jstl</groupId>
			<artifactId>jstl</artifactId>
			<version>${jstl.version}</version>
		</dependency>

		<!-- json lib -->
		<dependency>
			<groupId>net.sf.json-lib</groupId>
			<artifactId>json-lib</artifactId>
			<version>2.4</version>
			<classifier>jdk15</classifier>
		</dependency>

		<!-- mysql dependency -->
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<version>5.1.9</version>
		</dependency>

		<dependency>
			<groupId>org.slf4j</groupId>
			<artifactId>slf4j-log4j12</artifactId>
			<version>1.7.7</version>
		</dependency>


	</dependencies>
	<build>
		<finalName>spark-solr</finalName>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.1</version>
				<configuration>
					<source>1.7</source>
					<target>1.7</target>
					<encoding>UTF-8</encoding>
				</configuration>
			</plugin>
		</plugins>
	</build>
</project>

Now its time for to start with the configurations and let me remind since we are using Spring for as the core framework i will make use of Java configuration capability of Spring. so all my configurations are just java codes with out xml files. let us start with database config first
DBConfig.java

/**
 * 
 */
package com.spark.solr.web.config;

import javax.sql.DataSource;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DriverManagerDataSource;

/**
 * @author Sony
 *
 */
@Configuration
public class DBConfig {

	@Value("${db.driverClass}")
	String driverClassName;
	@Value("${db.url}")
	String url;
	@Value("${db.username}")
	String username;
	@Value("${db.password}")
	String password;
	
	@Bean
	public DataSource getDataSource(){
		DriverManagerDataSource driverManagerDataSource = new DriverManagerDataSource();
		driverManagerDataSource.setDriverClassName(driverClassName);
		driverManagerDataSource.setUrl(url);
		driverManagerDataSource.setUsername(username);
		driverManagerDataSource.setPassword(password);
		
		return driverManagerDataSource;
	}
	
	@Bean
	public JdbcTemplate getJdbcTemplate(DataSource dataSource){
		return new JdbcTemplate(dataSource);
	}
}

The above code is responsible for injecting the database properties and creating the datasource and finally injecting datasource into jdbc template class.
Now let us write the solr configuration integrated to spring.
SolrConfiguration.java

/**
 * 
 */
package com.spark.solr.web.config;

import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.web.client.RestTemplate;

/**
 * @author Sony
 *
 */
@Configuration
public class SolrConfiguration {

	@Value("${solr.server.host}")
	private String solrUrl;
	
	@Bean
	public SolrTemplate getSolrTemplate(SolrServer server){
		SolrTemplate solrTemplate = new SolrTemplate(server);
		return solrTemplate;
	}
	
	@Bean
	public SolrServer getSolrServer(){
		System.out.println("--------->"+solrUrl);
		SolrServer solrServer = new HttpSolrServer(solrUrl);
		return solrServer;
	}
	
}

Above class is responsible for creating the Solr Server object and injecting that to SolrTemplate class, using which we can perform operation over solr server. now to make our project web compatible lets code WebApplication initializer as below which acts similar to web.xml, this works only with servlet 3.0 container (tomcat7 and above)

/**
 * 
 */
package com.spark.solr.web.config;

import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.ServletRegistration.Dynamic;

import org.springframework.web.WebApplicationInitializer;
import org.springframework.web.context.support.AnnotationConfigWebApplicationContext;
import org.springframework.web.servlet.DispatcherServlet;

/**
 * @author Sony
 *
 */
public class WebAppInitializer implements WebApplicationInitializer{

	public void onStartup(ServletContext servletContext)
			throws ServletException {
		
		AnnotationConfigWebApplicationContext annotationConfigWebApplicationContext = new AnnotationConfigWebApplicationContext();
		annotationConfigWebApplicationContext.register(WebApplicationConfig.class);
		annotationConfigWebApplicationContext.setServletContext(servletContext);
		
		Dynamic dynamic = servletContext.addServlet("dispatcher", new DispatcherServlet(annotationConfigWebApplicationContext));
		dynamic.addMapping("/solr/*");
		dynamic.setLoadOnStartup(1);
		
		
	}

}

Below is the global config file to read the property files and to load other config files also the below code is responsible for detecting different annotations.

/** 
 * 
 */
package com.spark.solr.web.config;

import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.core.io.ClassPathResource;
import org.springframework.web.servlet.config.annotation.EnableWebMvc;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter;

/**
 * @author Sony
 *
 */
@Configuration
@EnableWebMvc
@ComponentScan(basePackages = "com.spark.solr")
@Import({ SolrConfiguration.class, DBConfig.class })
public class WebApplicationConfig extends WebMvcConfigurerAdapter {

	@Bean
	public PropertyPlaceholderConfigurer getPropertyPlaceholderConfigurer() {
		PropertyPlaceholderConfigurer placeholderConfigurer = new PropertyPlaceholderConfigurer();
		placeholderConfigurer.setLocation(new ClassPathResource(
				"application.properties"));
		placeholderConfigurer.setIgnoreUnresolvablePlaceholders(true);
		return placeholderConfigurer;
	}

}

Now the code concept of spring solr come here, Spring Data Solr, part of the larger Spring Data family, provides easy configuration and access to Apache Solr Search Server from Spring applications. It offers both low-level and high-level abstractions for interacting with the store.

Derived queries and annotations for Solr specific functionallity allow easy integration into existing applications. So spring data has provided Repositories to deal with different operations of Spring.

/**
 * 
 */
package com.spark.solr.web.repository;

import org.springframework.data.solr.repository.SolrCrudRepository;

import com.spark.solr.web.model.Entity;

/**
 * @author Sony
 *
 */
public interface EntityRepository extends SolrCrudRepository<Entity, Integer>{

}

Now let us start with our service layer and its implementation followed by DAO and its Implementation classes.
EntityService.java

/**
 * 
 */
package com.spark.solr.web.service;

import java.util.List;

/**
 * @author Sony
 *
 */
public interface EntityService {
	public void saveDocument();
	public void saveDocument(Object object);
	public void saveDocument(List<Object> objects);
}

EntityServiceImpl.java

/**
 * 
 */
package com.spark.solr.web.service.impl;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.data.solr.repository.support.SolrRepositoryFactory;
import org.springframework.stereotype.Service;

import com.spark.solr.web.dao.EntityDAO;
import com.spark.solr.web.model.Entity;
import com.spark.solr.web.repository.EntityRepository;
import com.spark.solr.web.service.EntityService;

/**
 * @author Sony
 *
 */
@Service(value = "entityServiceImpl")
public class EntityServiceImpl implements EntityService {

	@Autowired
	SolrTemplate solrTemplate;

	@Autowired
	EntityDAO entityDao;

	EntityRepository entityRepository;
	
	/* (non-Javadoc)
	 * @see com.spark.solr.web.service.EntityService#saveDocument(java.lang.Object)
	 */
	@Override
	public void saveDocument(Object object) {
		// TODO Auto-generated method stub

	}

	/* (non-Javadoc)
	 * @see com.spark.solr.web.service.EntityService#saveDocument(java.util.List)
	 */
	public void saveDocument(List<Object> objects) {
		// TODO Auto-generated method stub
	}
	
	/* (non-Javadoc)
	 * @see com.spark.solr.web.service.EntityService#saveDocument()
	 */
	@Override
	public void saveDocument() {
		getLogger().info("------------- Before fetching the details from database-----------");
		List<Map<String, Object>> values = entityDao.getDataFromDB();
		List<Entity> entities = new ArrayList<Entity>();
		
		for (Map<String, Object> map : values) {
			
			Entity entity = new Entity();
			
			entity.setEmployeeId(Integer.valueOf(map.get("EmployeeID").toString()));
			entity.setLastName(map.get("LastName").toString());
			entity.setFirstName(map.get("FirstName").toString());
			entity.setTitle(map.get("Title").toString());
			entity.setTitleOfCourtesy(map.get("TitleOfCourtesy").toString());
			entities.add(entity);
		}
		getLogger().info("------------- Before pushing documents to Solr server -----------");
		for (Entity entity : entities) {
			getEntityRepository().save(entity);
		}
		getLogger().info("------------- documents push to solr server done ! -----------");
	}
	
	/**
	 * @return
	 */
	public EntityRepository getEntityRepository(){
		return new SolrRepositoryFactory(solrTemplate.getSolrServer()).getRepository(EntityRepository.class);
	}

	public Logger getLogger(){
		return LoggerFactory.getLogger(EntityServiceImpl.class);
	}

}

EntityDAO.java

package com.spark.solr.web.dao;

import java.util.List;
import java.util.Map;


public interface EntityDAO {

	public List<Map<String, Object>> getDataFromDB();
}

EntityDAOImpl.java

/**
 * 
 */
package com.spark.solr.web.dao.impl;

import java.util.List;
import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;

import com.spark.solr.web.dao.EntityDAO;
import com.spark.solr.web.db.DBQueries;
import com.spark.solr.web.service.impl.EntityServiceImpl;

/**
 * @author Sony
 *
 */
@Repository(value="entityDao")
public class EntityDAOImpl implements EntityDAO {
	
	@Autowired
	private JdbcTemplate jdbcTemplate;

	/* (non-Javadoc)
	 * @see com.spark.solr.web.dao.EntityDAO#getDataFromDB()
	 */
	@Override
	public List<Map<String, Object>> getDataFromDB() {
		getLogger().info("----------- Querying database for the records -------------");
		getLogger().info("----------- sql query : "+DBQueries.GET_ALL_EMPLOYEES);
		List<Map<String, Object>> values = jdbcTemplate.queryForList(DBQueries.GET_ALL_EMPLOYEES);
		return values;
	}
	
	public Logger getLogger(){
		return LoggerFactory.getLogger(EntityDAOImpl.class);
	}

}

Now for making a better abstraction between the dao layer and the quires that are used in that layer i implemented a separate utility class as below.

/**
 * 
 */
package com.spark.solr.web.db;

/**
 * @author Sony
 *
 */
public class DBQueries {

	public static final String GET_ALL_EMPLOYEES = "select * from employee";
	
}

Finally our entity model looks as below.

/**
 * 
 */
package com.spark.solr.web.model;

import org.apache.solr.client.solrj.beans.Field;
import org.springframework.data.annotation.Id;

/**
 * @author Sony
 *
 */
public class Entity {

	@Id
	@Field(value="id")
	private int employeeId;
	@Field(value="lastName")
	private String lastName;
	@Field(value="firstName")
	private String firstName;
	@Field(value="title")
	private String title;
	@Field(value="titleOfCourtesy")
	private String titleOfCourtesy;

	/**
	 * @return the employeeId
	 */
	public int getEmployeeId() {
		return employeeId;
	}

	/**
	 * @param employeeId
	 *            the employeeId to set
	 */
	public void setEmployeeId(int employeeId) {
		this.employeeId = employeeId;
	}

	/**
	 * @return the lastName
	 */
	public String getLastName() {
		return lastName;
	}

	/**
	 * @param lastName
	 *            the lastName to set
	 */
	public void setLastName(String lastName) {
		this.lastName = lastName;
	}

	/**
	 * @return the firstName
	 */
	public String getFirstName() {
		return firstName;
	}

	/**
	 * @param firstName
	 *            the firstName to set
	 */
	public void setFirstName(String firstName) {
		this.firstName = firstName;
	}

	/**
	 * @return the title
	 */
	public String getTitle() {
		return title;
	}

	/**
	 * @param title
	 *            the title to set
	 */
	public void setTitle(String title) {
		this.title = title;
	}

	/**
	 * @return the titleOfCourtesy
	 */
	public String getTitleOfCourtesy() {
		return titleOfCourtesy;
	}

	/**
	 * @param titleOfCourtesy
	 *            the titleOfCourtesy to set
	 */
	public void setTitleOfCourtesy(String titleOfCourtesy) {
		this.titleOfCourtesy = titleOfCourtesy;
	}

}

Now let us look at the resource files that i used in this project 1.application.properties and 2.log4j.properties
please put the below files in “scr/main/resources” folder of your maven project.
application.properties

solr.server.host=http://localhost:8983/solr

db.driverClass=com.mysql.jdbc.Driver
db.url=jdbc:mysql://localhost:3306/test
db.username=root
db.password=root

log4j.properties

log4j.rootLogger = INFO, rollingFile

log4j.appender.rollingFile=org.apache.log4j.RollingFileAppender
log4j.appender.rollingFile.File=D:/spring-solr-dev.log
log4j.appender.rollingFile.MaxFileSize=2MB
log4j.appender.rollingFile.MaxBackupIndex=2
log4j.appender.rollingFile.layout = org.apache.log4j.PatternLayout
log4j.appender.rollingFile.layout.ConversionPattern=%p %t %c - %m%n

Note : please find the log file in d drive as i have given in log4j.properties file. please chage the drive location if you need.

Now we have done with the coding part, it is time for deployment, from cmd please navigate to location of project and issue the command as mvn clean install package this should build the project with all the required dependencies and final artifact would be spark-solr.war file.
now deploy this war file in apache tomcat under webapps directory and put the below url in your browser.
http://localhost:8080/spark-solr/solr/controller/load

Once we hit the url the controller calls the service layer and from service layer invoke dao layer to ftch the data from database abd returned back to service layer now in service layer we have the object od Solr repository and using that we push the data to solr.

After tha data is pushed we cah check tha data in solr using the solr admin page. as below
SolrAdmin

using my previous post we can pull the data from solr and show on custom UI.
Happy coding 🙂 happy Solr 🙂

JQGrid to display Apache Solr Data

Hi all, from past couple of weeks i was doing a POC on Solr in my project and there came a scenario where i need to pull the data from Solr server and display on JQGrid component. In this post let me show you how to do this task. The project structure looks as below.

jqgrid_project

let’s start with our index.html file as below.

<!DOCTYPE html>
<html>
<head>
<meta charset="ISO-8859-1">
<title>Solr Data Drid</title>
<link href="css/jquery-ui-custom.css" media="screen" type="text/css" rel="stylesheet">
<link href="css/ui.jqgrid.css" media="screen" type="text/css" rel="stylesheet">
<script type="text/javascript" src="js/jquery-1.9.0.min.js"></script>
<script type="text/javascript" src="js/grid.locale-en.js"></script>
<script type="text/javascript" src="js/jquery.jqGrid.min.js"></script>
<script type="text/javascript" src="custom_js/app.js"></script>
</head>
<body>
<table id="list2"></table>
<div id="pager2"></div>
</body>
</html>

Now comes the time for coding the app.js file where the actual logic reside. follow the comments carefully and the script is easy to understand.

/**
 * @author Pavan Kumar Mantha
 */

$(document)
		.ready(
				function() {
					var dataTable = $("#list2");
					var entityArray = [];

					dataTable.jqGrid({
						datatype : "local",
						colNames : [ 'id', 'lastName', 'firstName',
								'titleOfCourtesy' ],
						colModel : [ {
							name : 'id',
							index : 'id',
							width : 55
						}, {
							name : 'lastName',
							index : 'lastName',
							width : 90
						}, {
							name : 'firstName',
							index : 'firstName',
							width : 100
						}, {
							name : 'titleOfCourtesy',
							index : 'titleOfCourtesy',
							width : 80,
							align : "right"
						} ],
						rowNum : 10,
						rowList : [ 10, 20, 30 ],
						pager : '#pager2',
						sortname : 'id',
						viewrecords : true,
						sortorder : "desc",
						autowidth : true,
						caption : "Solr Data"
					});

					// handles the grid resize on window resize
					$(window).resize(
							function() {
								dataTable.jqGrid('setGridWidth', parseInt($(
										window).width()) - 20);
							});

					// Firing Ajax request to Solr server and getting data to
					// populate the jqgrid.
					// This ajax request is powered by cross domain
					$.ajaxSetup({
						crossDomain : true,
						scriptCharset : "utf-8",
						contentType : "jsonp; charset=utf-8"
					});

					var request = $
							.ajax({
								url : "http://localhost:8983/solr/collection1/select?q=*%3A*&wt=json&json.wrf=?&indent=true&rows=100",
								mthod : "GET",
								dataType : "jsonp"
							});

					request.done(function(data) {

						$.each(data.response.docs, function(i, doc) {
							var entityobject = {
								id : "",
								lastName : "",
								firstName : "",
								titleOfCourtesy : ""
							}

							entityobject.id = doc.id;
							entityobject.lastName = doc.lastName;
							entityobject.firstName = doc.firstName;
							entityobject.titleOfCourtesy = doc.titleOfCourtesy;

							entityArray.push(entityobject);
						});

						for (var i = 0; i <= entityArray.length; i++)
							jQuery("#list2").jqGrid('addRowData', i + 1,
									entityArray[i]);
					});

					request
							.fail(function(msg) {
								alert("Fetching of data from solr failed with error !");
							});
				});// end of (document).ready(function(){});


That is it, finally when we deploy this project to server and see the output it should look as below.
jqgrid_op

In my next post will be posting about the integration of SpringData and Apache Solr until then.
Happy coding jqgrid 🙂