1   /**
2    * Copyright (c) 2000-2010 Liferay, Inc. All rights reserved.
3    *
4    * The contents of this file are subject to the terms of the Liferay Enterprise
5    * Subscription License ("License"). You may not use this file except in
6    * compliance with the License. You can obtain a copy of the License by
7    * contacting Liferay, Inc. See the License for the specific language governing
8    * permissions and limitations under the License, including but not limited to
9    * distribution rights of the Software.
10   *
11   *
12   * 
13   */
14  
15  package com.liferay.portal.util;
16  
17  import com.liferay.portal.kernel.io.unsync.UnsyncByteArrayOutputStream;
18  import com.liferay.portal.kernel.log.Log;
19  import com.liferay.portal.kernel.log.LogFactoryUtil;
20  import com.liferay.portal.kernel.servlet.HttpHeaders;
21  import com.liferay.portal.kernel.util.ContentTypes;
22  import com.liferay.portal.kernel.util.FileUtil;
23  import com.liferay.portal.kernel.util.GetterUtil;
24  import com.liferay.portal.kernel.util.Http;
25  import com.liferay.portal.kernel.util.StringBundler;
26  import com.liferay.portal.kernel.util.StringPool;
27  import com.liferay.portal.kernel.util.StringUtil;
28  import com.liferay.portal.kernel.util.Validator;
29  import com.liferay.util.SystemProperties;
30  
31  import java.io.IOException;
32  import java.io.InputStream;
33  import java.io.UnsupportedEncodingException;
34  
35  import java.net.InetAddress;
36  import java.net.URL;
37  import java.net.URLConnection;
38  import java.net.URLDecoder;
39  import java.net.URLEncoder;
40  
41  import java.util.ArrayList;
42  import java.util.Date;
43  import java.util.LinkedHashMap;
44  import java.util.List;
45  import java.util.Map;
46  import java.util.StringTokenizer;
47  import java.util.regex.Pattern;
48  
49  import javax.portlet.ActionRequest;
50  import javax.portlet.RenderRequest;
51  
52  import javax.servlet.http.Cookie;
53  import javax.servlet.http.HttpServletRequest;
54  
55  import org.apache.commons.httpclient.Credentials;
56  import org.apache.commons.httpclient.Header;
57  import org.apache.commons.httpclient.HostConfiguration;
58  import org.apache.commons.httpclient.HttpClient;
59  import org.apache.commons.httpclient.HttpMethod;
60  import org.apache.commons.httpclient.HttpState;
61  import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
62  import org.apache.commons.httpclient.NTCredentials;
63  import org.apache.commons.httpclient.NameValuePair;
64  import org.apache.commons.httpclient.URI;
65  import org.apache.commons.httpclient.UsernamePasswordCredentials;
66  import org.apache.commons.httpclient.auth.AuthPolicy;
67  import org.apache.commons.httpclient.auth.AuthScope;
68  import org.apache.commons.httpclient.cookie.CookiePolicy;
69  import org.apache.commons.httpclient.methods.DeleteMethod;
70  import org.apache.commons.httpclient.methods.EntityEnclosingMethod;
71  import org.apache.commons.httpclient.methods.GetMethod;
72  import org.apache.commons.httpclient.methods.HeadMethod;
73  import org.apache.commons.httpclient.methods.PostMethod;
74  import org.apache.commons.httpclient.methods.PutMethod;
75  import org.apache.commons.httpclient.methods.RequestEntity;
76  import org.apache.commons.httpclient.methods.StringRequestEntity;
77  import org.apache.commons.httpclient.params.HttpClientParams;
78  import org.apache.commons.httpclient.params.HttpConnectionParams;
79  
80  /**
81   * <a href="HttpImpl.java.html"><b><i>View Source</i></b></a>
82   *
83   * @author Brian Wing Shun Chan
84   */
85  public class HttpImpl implements Http {
86  
87      public HttpImpl() {
88  
89          // Mimic behavior found in
90          // http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html
91  
92          if (Validator.isNotNull(_NON_PROXY_HOSTS)) {
93              String nonProxyHostsRegEx = _NON_PROXY_HOSTS;
94  
95              nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll(
96                  "\\.", "\\\\.");
97              nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll(
98                  "\\*", ".*?");
99              nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll(
100                 "\\|", ")|(");
101 
102             nonProxyHostsRegEx = "(" + nonProxyHostsRegEx + ")";
103 
104             _nonProxyHostsPattern = Pattern.compile(nonProxyHostsRegEx);
105         }
106 
107         MultiThreadedHttpConnectionManager httpConnectionManager =
108             new MultiThreadedHttpConnectionManager();
109 
110         HttpConnectionParams params = httpConnectionManager.getParams();
111 
112         params.setParameter(
113             "maxConnectionsPerHost", new Integer(_MAX_CONNECTIONS_PER_HOST));
114         params.setParameter(
115             "maxTotalConnections", new Integer(_MAX_TOTAL_CONNECTIONS));
116         params.setConnectionTimeout(_TIMEOUT);
117         params.setSoTimeout(_TIMEOUT);
118 
119         _client.setHttpConnectionManager(httpConnectionManager);
120         _proxyClient.setHttpConnectionManager(httpConnectionManager);
121 
122         if (hasProxyConfig() && Validator.isNotNull(_PROXY_USERNAME)) {
123             if (_PROXY_AUTH_TYPE.equals("username-password")) {
124                 _proxyCredentials = new UsernamePasswordCredentials(
125                     _PROXY_USERNAME, _PROXY_PASSWORD);
126             }
127             else if (_PROXY_AUTH_TYPE.equals("ntlm")) {
128                 _proxyCredentials = new NTCredentials(
129                     _PROXY_USERNAME, _PROXY_PASSWORD, _PROXY_NTLM_HOST,
130                     _PROXY_NTLM_DOMAIN);
131 
132                 List<String> authPrefs = new ArrayList<String>();
133 
134                 authPrefs.add(AuthPolicy.NTLM);
135                 authPrefs.add(AuthPolicy.BASIC);
136                 authPrefs.add(AuthPolicy.DIGEST);
137 
138                 _proxyClient.getParams().setParameter(
139                     AuthPolicy.AUTH_SCHEME_PRIORITY, authPrefs);
140             }
141         }
142     }
143 
144     public String addParameter(String url, String name, boolean value) {
145         return addParameter(url, name, String.valueOf(value));
146     }
147 
148     public String addParameter(String url, String name, double value) {
149         return addParameter(url, name, String.valueOf(value));
150     }
151 
152     public String addParameter(String url, String name, int value) {
153         return addParameter(url, name, String.valueOf(value));
154     }
155 
156     public String addParameter(String url, String name, long value) {
157         return addParameter(url, name, String.valueOf(value));
158     }
159 
160     public String addParameter(String url, String name, short value) {
161         return addParameter(url, name, String.valueOf(value));
162     }
163 
164     public String addParameter(String url, String name, String value) {
165         if (url == null) {
166             return null;
167         }
168 
169         String anchor = StringPool.BLANK;
170 
171         int pos = url.indexOf(StringPool.POUND);
172 
173         if (pos != -1) {
174             anchor = url.substring(pos);
175             url = url.substring(0, pos);
176         }
177 
178         if (url.indexOf(StringPool.QUESTION) == -1) {
179             url += StringPool.QUESTION;
180         }
181 
182         if (!url.endsWith(StringPool.QUESTION) &&
183             !url.endsWith(StringPool.AMPERSAND)) {
184 
185             url += StringPool.AMPERSAND;
186         }
187 
188         return url + name + StringPool.EQUAL + encodeURL(value) + anchor;
189     }
190 
191     public String decodeURL(String url) {
192         return decodeURL(url, false);
193     }
194 
195     public String decodeURL(String url, boolean unescapeSpace) {
196         if (url == null) {
197             return null;
198         }
199 
200         try {
201             url = URLDecoder.decode(url, StringPool.UTF8);
202 
203             if (unescapeSpace) {
204                 url = StringUtil.replace(url, "%20", StringPool.PLUS);
205             }
206 
207             return url;
208         }
209         catch (UnsupportedEncodingException uee) {
210             _log.error(uee, uee);
211 
212             return StringPool.BLANK;
213         }
214     }
215 
216     public void destroy() {
217         MultiThreadedHttpConnectionManager.shutdownAll();
218     }
219 
220     public String encodeURL(String url) {
221         return encodeURL(url, false);
222     }
223 
224     public String encodeURL(String url, boolean escapeSpaces) {
225         if (url == null) {
226             return null;
227         }
228 
229         try {
230             url = URLEncoder.encode(url, StringPool.UTF8);
231 
232             if (escapeSpaces) {
233                 url = StringUtil.replace(url, StringPool.PLUS, "%20");
234             }
235 
236             return url;
237         }
238         catch (UnsupportedEncodingException uee) {
239             _log.error(uee, uee);
240 
241             return StringPool.BLANK;
242         }
243     }
244 
245     public HttpClient getClient(HostConfiguration hostConfig) {
246         if (isProxyHost(hostConfig.getHost())) {
247             return _proxyClient;
248         }
249         else {
250             return _client;
251         }
252     }
253 
254     public String getCompleteURL(HttpServletRequest request) {
255         StringBuffer sb = request.getRequestURL();
256 
257         if (sb == null) {
258             sb = new StringBuffer();
259         }
260 
261         if (request.getQueryString() != null) {
262             sb.append(StringPool.QUESTION);
263             sb.append(request.getQueryString());
264         }
265 
266         String completeURL = sb.toString();
267 
268         if (_log.isWarnEnabled()) {
269             if (completeURL.contains("?&")) {
270                 _log.warn("Invalid url " + completeURL);
271             }
272         }
273 
274         return completeURL;
275     }
276 
277     public Cookie[] getCookies() {
278         return _cookies.get();
279     }
280 
281     public String getDomain(String url) {
282         url = removeProtocol(url);
283 
284         int pos = url.indexOf(StringPool.SLASH);
285 
286         if (pos != -1) {
287             return url.substring(0, pos);
288         }
289         else {
290             return url;
291         }
292     }
293 
294     public HostConfiguration getHostConfig(String location) throws IOException {
295         if (_log.isDebugEnabled()) {
296             _log.debug("Location is " + location);
297         }
298 
299         HostConfiguration hostConfig = new HostConfiguration();
300 
301         hostConfig.setHost(new URI(location, false));
302 
303         if (isProxyHost(hostConfig.getHost())) {
304             hostConfig.setProxy(_PROXY_HOST, _PROXY_PORT);
305         }
306 
307         return hostConfig;
308     }
309 
310     public String getIpAddress(String url) {
311         try {
312             URL urlObj = new URL(url);
313 
314             InetAddress address = InetAddress.getByName(urlObj.getHost());
315 
316             return address.getHostAddress();
317         }
318         catch (Exception e) {
319             return url;
320         }
321     }
322 
323     public String getParameter(String url, String name) {
324         return getParameter(url, name, true);
325     }
326 
327     public String getParameter(String url, String name, boolean escaped) {
328         if (Validator.isNull(url) || Validator.isNull(name)) {
329             return StringPool.BLANK;
330         }
331 
332         String[] parts = StringUtil.split(url, StringPool.QUESTION);
333 
334         if (parts.length == 2) {
335             String[] params = null;
336 
337             if (escaped) {
338                 params = StringUtil.split(parts[1], "&amp;");
339             }
340             else {
341                 params = StringUtil.split(parts[1], StringPool.AMPERSAND);
342             }
343 
344             for (int i = 0; i < params.length; i++) {
345                 String[] kvp = StringUtil.split(params[i], StringPool.EQUAL);
346 
347                 if ((kvp.length == 2) && kvp[0].equals(name)) {
348                     return kvp[1];
349                 }
350             }
351         }
352 
353         return StringPool.BLANK;
354     }
355 
356     public Map<String, String[]> getParameterMap(String queryString) {
357         return parameterMapFromString(queryString);
358     }
359 
360     public String getProtocol(ActionRequest actionRequest) {
361         return getProtocol(actionRequest.isSecure());
362     }
363 
364     public String getProtocol(boolean secure) {
365         if (!secure) {
366             return Http.HTTP;
367         }
368         else {
369             return Http.HTTPS;
370         }
371     }
372 
373     public String getProtocol(HttpServletRequest request) {
374         return getProtocol(request.isSecure());
375     }
376 
377     public String getProtocol(RenderRequest renderRequest) {
378         return getProtocol(renderRequest.isSecure());
379     }
380 
381     public String getProtocol(String url) {
382         int pos = url.indexOf(Http.PROTOCOL_DELIMITER);
383 
384         if (pos != -1) {
385             return url.substring(0, pos);
386         }
387         else {
388             return Http.HTTP;
389         }
390     }
391 
392     public String getQueryString(String url) {
393         if (Validator.isNull(url)) {
394             return url;
395         }
396 
397         int pos = url.indexOf(StringPool.QUESTION);
398 
399         if (pos == -1) {
400             return StringPool.BLANK;
401         }
402         else {
403             return url.substring(pos + 1, url.length());
404         }
405     }
406 
407     public String getRequestURL(HttpServletRequest request) {
408         return request.getRequestURL().toString();
409     }
410 
411     public boolean hasDomain(String url) {
412         return Validator.isNotNull(getDomain(url));
413     }
414 
415     public boolean hasProtocol(String url) {
416         int pos = url.indexOf(Http.PROTOCOL_DELIMITER);
417 
418         if (pos != -1) {
419             return true;
420         }
421         else {
422             return false;
423         }
424     }
425 
426     public boolean hasProxyConfig() {
427         if (Validator.isNotNull(_PROXY_HOST) && (_PROXY_PORT > 0)) {
428             return true;
429         }
430         else {
431             return false;
432         }
433     }
434 
435     public boolean isNonProxyHost(String host) {
436         if (_nonProxyHostsPattern == null ||
437             _nonProxyHostsPattern.matcher(host).matches()) {
438 
439             return true;
440         }
441         else {
442             return false;
443         }
444     }
445 
446     public boolean isProxyHost(String host) {
447         if (hasProxyConfig() && !isNonProxyHost(host)) {
448             return true;
449         }
450         else {
451             return false;
452         }
453     }
454 
455     public Map<String, String[]> parameterMapFromString(String queryString) {
456         Map<String, String[]> parameterMap =
457             new LinkedHashMap<String, String[]>();
458 
459         if (Validator.isNull(queryString)) {
460             return parameterMap;
461         }
462 
463         Map<String, List<String>> tempParameterMap =
464             new LinkedHashMap<String, List<String>>();
465 
466         StringTokenizer st = new StringTokenizer(
467             queryString, StringPool.AMPERSAND);
468 
469         while (st.hasMoreTokens()) {
470             String token = st.nextToken();
471 
472             if (Validator.isNotNull(token)) {
473                 String[] kvp = StringUtil.split(token, StringPool.EQUAL);
474 
475                 String key = kvp[0];
476 
477                 String value = StringPool.BLANK;
478 
479                 if (kvp.length > 1) {
480                     value = kvp[1];
481                 }
482 
483                 List<String> values = tempParameterMap.get(key);
484 
485                 if (values == null) {
486                     values = new ArrayList<String>();
487 
488                     tempParameterMap.put(key, values);
489                 }
490 
491                 values.add(value);
492             }
493         }
494 
495         for (Map.Entry<String, List<String>> entry :
496                 tempParameterMap.entrySet()) {
497 
498             String key = entry.getKey();
499             List<String> values = entry.getValue();
500 
501             parameterMap.put(key, values.toArray(new String[values.size()]));
502         }
503 
504         return parameterMap;
505     }
506 
507     public String parameterMapToString(Map<String, String[]> parameterMap) {
508         return parameterMapToString(parameterMap, true);
509     }
510 
511     public String parameterMapToString(
512         Map<String, String[]> parameterMap, boolean addQuestion) {
513 
514         StringBundler sb = new StringBundler();
515 
516         if (parameterMap.size() > 0) {
517             if (addQuestion) {
518                 sb.append(StringPool.QUESTION);
519             }
520 
521             for (Map.Entry<String, String[]> entry : parameterMap.entrySet()) {
522                 String name = entry.getKey();
523                 String[] values = entry.getValue();
524 
525                 for (String value : values) {
526                     sb.append(name);
527                     sb.append(StringPool.EQUAL);
528                     sb.append(encodeURL(value));
529                     sb.append(StringPool.AMPERSAND);
530                 }
531             }
532 
533             if (sb.index() > 1) {
534                 sb.setIndex(sb.index() - 1);
535             }
536         }
537 
538         return sb.toString();
539     }
540 
541     public String protocolize(String url, ActionRequest actionRequest) {
542         return protocolize(url, actionRequest.isSecure());
543     }
544 
545     public String protocolize(String url, boolean secure) {
546         if (secure) {
547             if (url.startsWith(Http.HTTP_WITH_SLASH)) {
548                 return StringUtil.replace(
549                     url, Http.HTTP_WITH_SLASH, Http.HTTPS_WITH_SLASH);
550             }
551         }
552         else {
553             if (url.startsWith(Http.HTTPS_WITH_SLASH)) {
554                 return StringUtil.replace(
555                     url, Http.HTTPS_WITH_SLASH, Http.HTTP_WITH_SLASH);
556             }
557         }
558 
559         return url;
560     }
561 
562     public String protocolize(String url, HttpServletRequest request) {
563         return protocolize(url, request.isSecure());
564     }
565 
566     public String protocolize(String url, RenderRequest renderRequest) {
567         return protocolize(url, renderRequest.isSecure());
568     }
569 
570     public String removeDomain(String url) {
571         url = removeProtocol(url);
572 
573         int pos = url.indexOf(StringPool.SLASH);
574 
575         if (pos > 0) {
576             return url.substring(pos);
577         }
578         else {
579             return url;
580         }
581     }
582 
583     public String removeParameter(String url, String name) {
584         int pos = url.indexOf(StringPool.QUESTION);
585 
586         if (pos == -1) {
587             return url;
588         }
589 
590         String anchor = StringPool.BLANK;
591 
592         int anchorPos = url.indexOf(StringPool.POUND);
593 
594         if (anchorPos != -1) {
595             anchor = url.substring(anchorPos);
596             url = url.substring(0, anchorPos);
597         }
598 
599         StringBundler sb = new StringBundler();
600 
601         sb.append(url.substring(0, pos + 1));
602 
603         StringTokenizer st = new StringTokenizer(
604             url.substring(pos + 1, url.length()), StringPool.AMPERSAND);
605 
606         while (st.hasMoreTokens()) {
607             String token = st.nextToken();
608 
609             if (Validator.isNotNull(token)) {
610                 String[] kvp = StringUtil.split(token, StringPool.EQUAL);
611 
612                 String key = kvp[0];
613 
614                 String value = StringPool.BLANK;
615 
616                 if (kvp.length > 1) {
617                     value = kvp[1];
618                 }
619 
620                 if (!key.equals(name)) {
621                     sb.append(key);
622                     sb.append(StringPool.EQUAL);
623                     sb.append(value);
624                     sb.append(StringPool.AMPERSAND);
625                 }
626             }
627         }
628 
629         url = StringUtil.replace(
630             sb.toString(), StringPool.AMPERSAND + StringPool.AMPERSAND,
631             StringPool.AMPERSAND);
632 
633         if (url.endsWith(StringPool.AMPERSAND)) {
634             url = url.substring(0, url.length() - 1);
635         }
636 
637         if (url.endsWith(StringPool.QUESTION)) {
638             url = url.substring(0, url.length() - 1);
639         }
640 
641         return url + anchor;
642     }
643 
644     public String removeProtocol(String url) {
645         if (url.startsWith(Http.HTTP_WITH_SLASH)) {
646             return url.substring(Http.HTTP_WITH_SLASH.length() , url.length());
647         }
648         else if (url.startsWith(Http.HTTPS_WITH_SLASH)) {
649             return url.substring(Http.HTTPS_WITH_SLASH.length() , url.length());
650         }
651         else {
652             return url;
653         }
654     }
655 
656     public String setParameter(String url, String name, boolean value) {
657         return setParameter(url, name, String.valueOf(value));
658     }
659 
660     public String setParameter(String url, String name, double value) {
661         return setParameter(url, name, String.valueOf(value));
662     }
663 
664     public String setParameter(String url, String name, int value) {
665         return setParameter(url, name, String.valueOf(value));
666     }
667 
668     public String setParameter(String url, String name, long value) {
669         return setParameter(url, name, String.valueOf(value));
670     }
671 
672     public String setParameter(String url, String name, short value) {
673         return setParameter(url, name, String.valueOf(value));
674     }
675 
676     public String setParameter(String url, String name, String value) {
677         if (url == null) {
678             return null;
679         }
680 
681         url = removeParameter(url, name);
682 
683         return addParameter(url, name, value);
684     }
685 
686     /**
687      * @deprecated
688      */
689     public void submit(String location) throws IOException {
690         URLtoByteArray(location);
691     }
692 
693     /**
694      * @deprecated
695      */
696     public void submit(String location, boolean post) throws IOException {
697         URLtoByteArray(location, post);
698     }
699 
700     /**
701      * @deprecated
702      */
703     public void submit(String location, Cookie[] cookies) throws IOException {
704         URLtoByteArray(location, cookies);
705     }
706 
707     /**
708      * @deprecated
709      */
710     public void submit(String location, Cookie[] cookies, boolean post)
711         throws IOException {
712 
713         URLtoByteArray(location, cookies, post);
714     }
715 
716     /**
717      * @deprecated
718      */
719     public void submit(
720             String location, Cookie[] cookies, Http.Body body, boolean post)
721         throws IOException {
722 
723         URLtoByteArray(location, cookies, body, post);
724     }
725 
726     /**
727      * @deprecated
728      */
729     public void submit(
730             String location, Cookie[] cookies, Map<String, String> parts,
731             boolean post)
732         throws IOException {
733 
734         URLtoByteArray(location, cookies, parts, post);
735     }
736 
737     public byte[] URLtoByteArray(Http.Options options) throws IOException {
738         return URLtoByteArray(
739             options.getLocation(), options.getMethod(), options.getHeaders(),
740             options.getCookies(), options.getAuth(), options.getBody(),
741             options.getParts(), options.getResponse(),
742             options.isFollowRedirects());
743     }
744 
745     public byte[] URLtoByteArray(String location) throws IOException {
746         Http.Options options = new Http.Options();
747 
748         options.setLocation(location);
749 
750         return URLtoByteArray(options);
751     }
752 
753     public byte[] URLtoByteArray(String location, boolean post)
754         throws IOException {
755 
756         Http.Options options = new Http.Options();
757 
758         options.setLocation(location);
759         options.setPost(post);
760 
761         return URLtoByteArray(options);
762     }
763 
764     /**
765      * @deprecated
766      */
767     public byte[] URLtoByteArray(String location, Cookie[] cookies)
768         throws IOException {
769 
770         Http.Options options = new Http.Options();
771 
772         options.setCookies(cookies);
773         options.setLocation(location);
774 
775         return URLtoByteArray(options);
776     }
777 
778     /**
779      * @deprecated
780      */
781     public byte[] URLtoByteArray(
782             String location, Cookie[] cookies, boolean post)
783         throws IOException {
784 
785         Http.Options options = new Http.Options();
786 
787         options.setCookies(cookies);
788         options.setLocation(location);
789         options.setPost(post);
790 
791         return URLtoByteArray(options);
792     }
793 
794     /**
795      * @deprecated
796      */
797     public byte[] URLtoByteArray(
798             String location, Cookie[] cookies, Http.Auth auth, Http.Body body,
799             boolean post)
800         throws IOException {
801 
802         Http.Options options = new Http.Options();
803 
804         options.setAuth(auth);
805         options.setBody(body);
806         options.setCookies(cookies);
807         options.setLocation(location);
808         options.setPost(post);
809 
810         return URLtoByteArray(options);
811     }
812 
813     /**
814      * @deprecated
815      */
816     public byte[] URLtoByteArray(
817             String location, Cookie[] cookies, Http.Auth auth,
818             Map<String, String> parts, boolean post)
819         throws IOException {
820 
821         Http.Options options = new Http.Options();
822 
823         options.setAuth(auth);
824         options.setCookies(cookies);
825         options.setLocation(location);
826         options.setParts(parts);
827         options.setPost(post);
828 
829         return URLtoByteArray(options);
830     }
831 
832     /**
833      * @deprecated
834      */
835     public byte[] URLtoByteArray(
836             String location, Cookie[] cookies, Http.Body body, boolean post)
837         throws IOException {
838 
839         Http.Options options = new Http.Options();
840 
841         options.setBody(body);
842         options.setCookies(cookies);
843         options.setLocation(location);
844         options.setPost(post);
845 
846         return URLtoByteArray(options);
847     }
848 
849     /**
850      * @deprecated
851      */
852     public byte[] URLtoByteArray(
853             String location, Cookie[] cookies, Map<String, String> parts,
854             boolean post)
855         throws IOException {
856 
857         Http.Options options = new Http.Options();
858 
859         options.setCookies(cookies);
860         options.setLocation(location);
861         options.setParts(parts);
862         options.setPost(post);
863 
864         return URLtoByteArray(options);
865     }
866 
867     public String URLtoString(Http.Options options) throws IOException {
868         return new String(URLtoByteArray(options));
869     }
870 
871     public String URLtoString(String location) throws IOException {
872         return new String(URLtoByteArray(location));
873     }
874 
875     public String URLtoString(String location, boolean post)
876         throws IOException {
877 
878         return new String(URLtoByteArray(location, post));
879     }
880 
881     /**
882      * @deprecated
883      */
884     public String URLtoString(String location, Cookie[] cookies)
885         throws IOException {
886 
887         return new String(URLtoByteArray(location, cookies));
888     }
889 
890     /**
891      * @deprecated
892      */
893     public String URLtoString(
894             String location, Cookie[] cookies, boolean post)
895         throws IOException {
896 
897         return new String(URLtoByteArray(location, cookies, post));
898     }
899 
900     /**
901      * @deprecated
902      */
903     public String URLtoString(
904             String location, Cookie[] cookies, Http.Auth auth, Http.Body body,
905             boolean post)
906         throws IOException {
907 
908         return new String(URLtoByteArray(location, cookies, auth, body, post));
909     }
910 
911     /**
912      * @deprecated
913      */
914     public String URLtoString(
915             String location, Cookie[] cookies, Http.Auth auth,
916             Map<String, String> parts, boolean post)
917         throws IOException {
918 
919         return new String(URLtoByteArray(location, cookies, auth, parts, post));
920     }
921 
922     /**
923      * @deprecated
924      */
925     public String URLtoString(
926             String location, Cookie[] cookies, Http.Body body, boolean post)
927         throws IOException {
928 
929         return new String(URLtoByteArray(location, cookies, body, post));
930     }
931 
932     /**
933      * @deprecated
934      */
935     public String URLtoString(
936             String location, Cookie[] cookies, Map<String, String> parts,
937             boolean post)
938         throws IOException {
939 
940         return new String(URLtoByteArray(location, cookies, null, parts, post));
941     }
942 
943     /**
944      * @deprecated
945      */
946     public String URLtoString(
947             String location, String host, int port, String realm,
948             String username, String password)
949         throws IOException {
950 
951         return URLtoString(
952             location, (Cookie[])null,
953             new Http.Auth(host, port, realm, username, password),
954             (Http.Body)null, false);
955     }
956 
957     /**
958      * This method only uses the default Commons HttpClient implementation when
959      * the URL object represents a HTTP resource. The URL object could also
960      * represent a file or some JNDI resource. In that case, the default Java
961      * implementation is used.
962      *
963      * @param  url URL object
964      * @return A string representation of the resource referenced by the
965      *         URL object
966      */
967     public String URLtoString(URL url) throws IOException {
968         String xml = null;
969 
970         if (url != null) {
971             String protocol = url.getProtocol().toLowerCase();
972 
973             if (protocol.startsWith(Http.HTTP) ||
974                 protocol.startsWith(Http.HTTPS)) {
975 
976                 return URLtoString(url.toString());
977             }
978 
979             URLConnection con = url.openConnection();
980 
981             InputStream is = con.getInputStream();
982 
983             UnsyncByteArrayOutputStream ubaos =
984                 new UnsyncByteArrayOutputStream();
985             byte[] bytes = new byte[512];
986 
987             for (int i = is.read(bytes, 0, 512); i != -1;
988                     i = is.read(bytes, 0, 512)) {
989 
990                 ubaos.write(bytes, 0, i);
991             }
992 
993             xml = new String(ubaos.unsafeGetByteArray(), 0, ubaos.size());
994 
995             is.close();
996             ubaos.close();
997         }
998 
999         return xml;
1000    }
1001
1002    protected void proxifyState(HttpState state, HostConfiguration hostConfig) {
1003        Credentials proxyCredentials = _proxyCredentials;
1004
1005        String host = hostConfig.getHost();
1006
1007        if (isProxyHost(host) && (proxyCredentials != null)) {
1008            AuthScope scope = new AuthScope(_PROXY_HOST, _PROXY_PORT, null);
1009
1010            state.setProxyCredentials(scope, proxyCredentials);
1011        }
1012    }
1013
1014    protected org.apache.commons.httpclient.Cookie toCommonsCookie(
1015        Cookie cookie) {
1016
1017        org.apache.commons.httpclient.Cookie commonsCookie =
1018            new org.apache.commons.httpclient.Cookie(
1019            cookie.getDomain(), cookie.getName(), cookie.getValue(),
1020            cookie.getPath(), cookie.getMaxAge(), cookie.getSecure());
1021
1022        commonsCookie.setVersion(cookie.getVersion());
1023
1024        return commonsCookie;
1025    }
1026
1027    protected org.apache.commons.httpclient.Cookie[] toCommonsCookies(
1028        Cookie[] cookies) {
1029
1030        if (cookies == null) {
1031            return null;
1032        }
1033
1034        org.apache.commons.httpclient.Cookie[] commonCookies =
1035            new org.apache.commons.httpclient.Cookie[cookies.length];
1036
1037        for (int i = 0; i < cookies.length; i++) {
1038            commonCookies[i] = toCommonsCookie(cookies[i]);
1039        }
1040
1041        return commonCookies;
1042    }
1043
1044    protected Cookie toServletCookie(
1045        org.apache.commons.httpclient.Cookie commonsCookie) {
1046
1047        Cookie cookie = new Cookie(
1048            commonsCookie.getName(), commonsCookie.getValue());
1049
1050        cookie.setDomain(commonsCookie.getDomain());
1051
1052        Date expiryDate = commonsCookie.getExpiryDate();
1053
1054        if (expiryDate != null) {
1055            int maxAge =
1056                (int)(expiryDate.getTime() - System.currentTimeMillis());
1057
1058            maxAge = maxAge / 1000;
1059
1060            if (maxAge > -1) {
1061                cookie.setMaxAge(maxAge);
1062            }
1063        }
1064
1065        cookie.setPath(commonsCookie.getPath());
1066        cookie.setSecure(commonsCookie.getSecure());
1067        cookie.setVersion(commonsCookie.getVersion());
1068
1069        return cookie;
1070    }
1071
1072    protected Cookie[] toServletCookies(
1073        org.apache.commons.httpclient.Cookie[] commonsCookies) {
1074
1075        if (commonsCookies == null) {
1076            return null;
1077        }
1078
1079        Cookie[] cookies = new Cookie[commonsCookies.length];
1080
1081        for (int i = 0; i < commonsCookies.length; i++) {
1082            cookies[i] = toServletCookie(commonsCookies[i]);
1083        }
1084
1085        return cookies;
1086    }
1087
1088    protected byte[] URLtoByteArray(
1089            String location, Http.Method method, Map<String, String> headers,
1090            Cookie[] cookies, Http.Auth auth, Http.Body body, Map<String,
1091            String> parts, Http.Response response, boolean followRedirects)
1092        throws IOException {
1093
1094        byte[] bytes = null;
1095
1096        HttpMethod httpMethod = null;
1097        HttpState httpState = null;
1098
1099        try {
1100            _cookies.set(null);
1101
1102            if (location == null) {
1103                return bytes;
1104            }
1105            else if (!location.startsWith(Http.HTTP_WITH_SLASH) &&
1106                     !location.startsWith(Http.HTTPS_WITH_SLASH)) {
1107
1108                location = Http.HTTP_WITH_SLASH + location;
1109            }
1110
1111            HostConfiguration hostConfig = getHostConfig(location);
1112
1113            HttpClient httpClient = getClient(hostConfig);
1114
1115            if ((method == Http.Method.POST) ||
1116                (method == Http.Method.PUT)) {
1117
1118                if (method == Http.Method.POST) {
1119                    httpMethod = new PostMethod(location);
1120                }
1121                else {
1122                    httpMethod = new PutMethod(location);
1123                }
1124
1125                if (body != null) {
1126                    RequestEntity requestEntity = new StringRequestEntity(
1127                        body.getContent(), body.getContentType(),
1128                        body.getCharset());
1129
1130                    EntityEnclosingMethod entityEnclosingMethod =
1131                        (EntityEnclosingMethod)httpMethod;
1132
1133                    entityEnclosingMethod.setRequestEntity(requestEntity);
1134                }
1135                else if ((parts != null) && (parts.size() > 0) &&
1136                         (method == Http.Method.POST)) {
1137
1138                    List<NameValuePair> nvpList =
1139                        new ArrayList<NameValuePair>();
1140
1141                    for (Map.Entry<String, String> entry : parts.entrySet()) {
1142                        String key = entry.getKey();
1143                        String value = entry.getValue();
1144
1145                        if (value != null) {
1146                            nvpList.add(new NameValuePair(key, value));
1147                        }
1148                    }
1149
1150                    NameValuePair[] nvpArray = nvpList.toArray(
1151                        new NameValuePair[nvpList.size()]);
1152
1153                    PostMethod postMethod = (PostMethod)httpMethod;
1154
1155                    postMethod.setRequestBody(nvpArray);
1156                }
1157            }
1158            else if (method == Http.Method.DELETE) {
1159                httpMethod = new DeleteMethod(location);
1160            }
1161            else if (method == Http.Method.HEAD) {
1162                httpMethod = new HeadMethod(location);
1163            }
1164            else {
1165                httpMethod = new GetMethod(location);
1166            }
1167
1168            if (headers != null) {
1169                for (Map.Entry<String, String> header : headers.entrySet()) {
1170                    httpMethod.addRequestHeader(
1171                        header.getKey(), header.getValue());
1172                }
1173            }
1174
1175            if ((method == Http.Method.POST) || (method == Http.Method.PUT) &&
1176                (body != null)) {
1177            }
1178            else if (!_hasRequestHeader(httpMethod, HttpHeaders.CONTENT_TYPE)) {
1179                httpMethod.addRequestHeader(
1180                    HttpHeaders.CONTENT_TYPE,
1181                    ContentTypes.APPLICATION_X_WWW_FORM_URLENCODED);
1182            }
1183
1184            if (!_hasRequestHeader(httpMethod, HttpHeaders.USER_AGENT)) {
1185                httpMethod.addRequestHeader(
1186                    HttpHeaders.USER_AGENT, _DEFAULT_USER_AGENT);
1187            }
1188
1189            httpMethod.getParams().setIntParameter(
1190                HttpClientParams.SO_TIMEOUT, 0);
1191
1192            httpState = new HttpState();
1193
1194            if ((cookies != null) && (cookies.length > 0)) {
1195                org.apache.commons.httpclient.Cookie[] commonsCookies =
1196                    toCommonsCookies(cookies);
1197
1198                httpState.addCookies(commonsCookies);
1199
1200                httpMethod.getParams().setCookiePolicy(
1201                    CookiePolicy.BROWSER_COMPATIBILITY);
1202            }
1203
1204            if (auth != null) {
1205                httpMethod.setDoAuthentication(true);
1206
1207                httpState.setCredentials(
1208                    new AuthScope(
1209                        auth.getHost(), auth.getPort(), auth.getRealm()),
1210                    new UsernamePasswordCredentials(
1211                        auth.getUsername(), auth.getPassword()));
1212            }
1213
1214            proxifyState(httpState, hostConfig);
1215
1216            httpClient.executeMethod(hostConfig, httpMethod, httpState);
1217
1218            Header locationHeader = httpMethod.getResponseHeader("location");
1219
1220            if ((locationHeader != null) && !locationHeader.equals(location)) {
1221                String redirect = locationHeader.getValue();
1222
1223                if (followRedirects) {
1224                    return URLtoByteArray(
1225                        redirect, Http.Method.GET, headers,
1226                        cookies, auth, body, parts, response, followRedirects);
1227                }
1228                else {
1229                    response.setRedirect(redirect);
1230                }
1231            }
1232
1233            InputStream is = httpMethod.getResponseBodyAsStream();
1234
1235            if (is != null) {
1236                Header contentLength = httpMethod.getResponseHeader(
1237                    HttpHeaders.CONTENT_LENGTH);
1238
1239                if (contentLength != null) {
1240                    response.setContentLength(
1241                        GetterUtil.getInteger(contentLength.getValue()));
1242                }
1243
1244                Header contentType = httpMethod.getResponseHeader(
1245                    HttpHeaders.CONTENT_TYPE);
1246
1247                if (contentType != null) {
1248                    response.setContentType(contentType.getValue());
1249                }
1250
1251                bytes = FileUtil.getBytes(is);
1252
1253                is.close();
1254            }
1255
1256            for (Header header : httpMethod.getResponseHeaders()) {
1257                response.addHeader(header.getName(), header.getValue());
1258            }
1259
1260            return bytes;
1261        }
1262        finally {
1263            try {
1264                if (httpState != null) {
1265                    _cookies.set(toServletCookies(httpState.getCookies()));
1266                }
1267            }
1268            catch (Exception e) {
1269                _log.error(e, e);
1270            }
1271
1272            try {
1273                if (httpMethod != null) {
1274                    httpMethod.releaseConnection();
1275                }
1276            }
1277            catch (Exception e) {
1278                _log.error(e, e);
1279            }
1280        }
1281    }
1282
1283    private boolean _hasRequestHeader(HttpMethod httpMethod, String name) {
1284        if (httpMethod.getRequestHeaders(name).length == 0) {
1285            return false;
1286        }
1287        else {
1288            return true;
1289        }
1290    }
1291
1292    private static final String _DEFAULT_USER_AGENT =
1293        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
1294
1295    private static final int _MAX_CONNECTIONS_PER_HOST = GetterUtil.getInteger(
1296        PropsUtil.get(HttpImpl.class.getName() + ".max.connections.per.host"),
1297        2);
1298
1299    private static final int _MAX_TOTAL_CONNECTIONS = GetterUtil.getInteger(
1300        PropsUtil.get(HttpImpl.class.getName() + ".max.total.connections"),
1301        20);
1302
1303    private static final String _NON_PROXY_HOSTS =
1304        SystemProperties.get("http.nonProxyHosts");
1305
1306    private static final String _PROXY_AUTH_TYPE = GetterUtil.getString(
1307        PropsUtil.get(HttpImpl.class.getName() + ".proxy.auth.type"));
1308
1309    private static final String _PROXY_HOST = GetterUtil.getString(
1310        SystemProperties.get("http.proxyHost"));
1311
1312    private static final String _PROXY_NTLM_DOMAIN = GetterUtil.getString(
1313        PropsUtil.get(HttpImpl.class.getName() + ".proxy.ntlm.domain"));
1314
1315    private static final String _PROXY_NTLM_HOST = GetterUtil.getString(
1316        PropsUtil.get(HttpImpl.class.getName() + ".proxy.ntlm.host"));
1317
1318    private static final String _PROXY_PASSWORD = GetterUtil.getString(
1319        PropsUtil.get(HttpImpl.class.getName() + ".proxy.password"));
1320
1321    private static final int _PROXY_PORT = GetterUtil.getInteger(
1322        SystemProperties.get("http.proxyPort"));
1323
1324    private static final String _PROXY_USERNAME = GetterUtil.getString(
1325        PropsUtil.get(HttpImpl.class.getName() + ".proxy.username"));
1326
1327    private static final int _TIMEOUT = GetterUtil.getInteger(
1328        PropsUtil.get(HttpImpl.class.getName() + ".timeout"), 5000);
1329
1330    private static Log _log = LogFactoryUtil.getLog(HttpImpl.class);
1331
1332    private static ThreadLocal<Cookie[]> _cookies = new ThreadLocal<Cookie[]>();
1333
1334    private HttpClient _client = new HttpClient();
1335    private Pattern _nonProxyHostsPattern;
1336    private HttpClient _proxyClient = new HttpClient();
1337    private Credentials _proxyCredentials;
1338
1339}