Support for Robots Exclusion Protocol, including parsing and matching against robot.txt directives.
'com.brandwatch:robots-core:1.1.2'
<dependency>
<groupId>com.brandwatch</groupId>
<artifactId>robots-core</artifactId>
<version>1.1.2</version>
</dependency>
<dependency org="com.brandwatch" name="robots-core" rev="1.1.2"/>
"com.brandwatch", "robots-core", "1.1.2"